Merge branch 'rs/xdiff-ignore-ws-w-func-context'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 # support basestring in python3
31 try:
32     unicode = unicode
33 except NameError:
34     # 'unicode' is undefined, must be Python 3
35     str = str
36     unicode = str
37     bytes = bytes
38     basestring = (str,bytes)
39 else:
40     # 'unicode' exists, must be Python 2
41     str = str
42     unicode = unicode
43     bytes = str
44     basestring = basestring
45
46 try:
47     from subprocess import CalledProcessError
48 except ImportError:
49     # from python2.7:subprocess.py
50     # Exception classes used by this module.
51     class CalledProcessError(Exception):
52         """This exception is raised when a process run by check_call() returns
53         a non-zero exit status.  The exit status will be stored in the
54         returncode attribute."""
55         def __init__(self, returncode, cmd):
56             self.returncode = returncode
57             self.cmd = cmd
58         def __str__(self):
59             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
61 verbose = False
62
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
65
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
68
69 p4_access_checked = False
70
71 def p4_build_cmd(cmd):
72     """Build a suitable p4 command line.
73
74     This consolidates building and returning a p4 command line into one
75     location. It means that hooking into the environment, or other configuration
76     can be done more easily.
77     """
78     real_cmd = ["p4"]
79
80     user = gitConfig("git-p4.user")
81     if len(user) > 0:
82         real_cmd += ["-u",user]
83
84     password = gitConfig("git-p4.password")
85     if len(password) > 0:
86         real_cmd += ["-P", password]
87
88     port = gitConfig("git-p4.port")
89     if len(port) > 0:
90         real_cmd += ["-p", port]
91
92     host = gitConfig("git-p4.host")
93     if len(host) > 0:
94         real_cmd += ["-H", host]
95
96     client = gitConfig("git-p4.client")
97     if len(client) > 0:
98         real_cmd += ["-c", client]
99
100     retries = gitConfigInt("git-p4.retries")
101     if retries is None:
102         # Perform 3 retries by default
103         retries = 3
104     if retries > 0:
105         # Provide a way to not pass this option by setting git-p4.retries to 0
106         real_cmd += ["-r", str(retries)]
107
108     if isinstance(cmd,basestring):
109         real_cmd = ' '.join(real_cmd) + ' ' + cmd
110     else:
111         real_cmd += cmd
112
113     # now check that we can actually talk to the server
114     global p4_access_checked
115     if not p4_access_checked:
116         p4_access_checked = True    # suppress access checks in p4_check_access itself
117         p4_check_access()
118
119     return real_cmd
120
121 def git_dir(path):
122     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123         This won't automatically add ".git" to a directory.
124     """
125     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126     if not d or len(d) == 0:
127         return None
128     else:
129         return d
130
131 def chdir(path, is_client_path=False):
132     """Do chdir to the given path, and set the PWD environment
133        variable for use by P4.  It does not look at getcwd() output.
134        Since we're not using the shell, it is necessary to set the
135        PWD environment variable explicitly.
136
137        Normally, expand the path to force it to be absolute.  This
138        addresses the use of relative path names inside P4 settings,
139        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
140        as given; it looks for .p4config using PWD.
141
142        If is_client_path, the path was handed to us directly by p4,
143        and may be a symbolic link.  Do not call os.getcwd() in this
144        case, because it will cause p4 to think that PWD is not inside
145        the client path.
146        """
147
148     os.chdir(path)
149     if not is_client_path:
150         path = os.getcwd()
151     os.environ['PWD'] = path
152
153 def calcDiskFree():
154     """Return free space in bytes on the disk of the given dirname."""
155     if platform.system() == 'Windows':
156         free_bytes = ctypes.c_ulonglong(0)
157         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158         return free_bytes.value
159     else:
160         st = os.statvfs(os.getcwd())
161         return st.f_bavail * st.f_frsize
162
163 def die(msg):
164     if verbose:
165         raise Exception(msg)
166     else:
167         sys.stderr.write(msg + "\n")
168         sys.exit(1)
169
170 def write_pipe(c, stdin):
171     if verbose:
172         sys.stderr.write('Writing pipe: %s\n' % str(c))
173
174     expand = isinstance(c,basestring)
175     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176     pipe = p.stdin
177     val = pipe.write(stdin)
178     pipe.close()
179     if p.wait():
180         die('Command failed: %s' % str(c))
181
182     return val
183
184 def p4_write_pipe(c, stdin):
185     real_cmd = p4_build_cmd(c)
186     return write_pipe(real_cmd, stdin)
187
188 def read_pipe_full(c):
189     """ Read output from  command. Returns a tuple
190         of the return status, stdout text and stderr
191         text.
192     """
193     if verbose:
194         sys.stderr.write('Reading pipe: %s\n' % str(c))
195
196     expand = isinstance(c,basestring)
197     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198     (out, err) = p.communicate()
199     return (p.returncode, out, err)
200
201 def read_pipe(c, ignore_error=False):
202     """ Read output from  command. Returns the output text on
203         success. On failure, terminates execution, unless
204         ignore_error is True, when it returns an empty string.
205     """
206     (retcode, out, err) = read_pipe_full(c)
207     if retcode != 0:
208         if ignore_error:
209             out = ""
210         else:
211             die('Command failed: %s\nError: %s' % (str(c), err))
212     return out
213
214 def read_pipe_text(c):
215     """ Read output from a command with trailing whitespace stripped.
216         On error, returns None.
217     """
218     (retcode, out, err) = read_pipe_full(c)
219     if retcode != 0:
220         return None
221     else:
222         return out.rstrip()
223
224 def p4_read_pipe(c, ignore_error=False):
225     real_cmd = p4_build_cmd(c)
226     return read_pipe(real_cmd, ignore_error)
227
228 def read_pipe_lines(c):
229     if verbose:
230         sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232     expand = isinstance(c, basestring)
233     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234     pipe = p.stdout
235     val = pipe.readlines()
236     if pipe.close() or p.wait():
237         die('Command failed: %s' % str(c))
238
239     return val
240
241 def p4_read_pipe_lines(c):
242     """Specifically invoke p4 on the command supplied. """
243     real_cmd = p4_build_cmd(c)
244     return read_pipe_lines(real_cmd)
245
246 def p4_has_command(cmd):
247     """Ask p4 for help on this command.  If it returns an error, the
248        command does not exist in this version of p4."""
249     real_cmd = p4_build_cmd(["help", cmd])
250     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251                                    stderr=subprocess.PIPE)
252     p.communicate()
253     return p.returncode == 0
254
255 def p4_has_move_command():
256     """See if the move command exists, that it supports -k, and that
257        it has not been administratively disabled.  The arguments
258        must be correct, but the filenames do not have to exist.  Use
259        ones with wildcards so even if they exist, it will fail."""
260
261     if not p4_has_command("move"):
262         return False
263     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265     (out, err) = p.communicate()
266     # return code will be 1 in either case
267     if err.find("Invalid option") >= 0:
268         return False
269     if err.find("disabled") >= 0:
270         return False
271     # assume it failed because @... was invalid changelist
272     return True
273
274 def system(cmd, ignore_error=False):
275     expand = isinstance(cmd,basestring)
276     if verbose:
277         sys.stderr.write("executing %s\n" % str(cmd))
278     retcode = subprocess.call(cmd, shell=expand)
279     if retcode and not ignore_error:
280         raise CalledProcessError(retcode, cmd)
281
282     return retcode
283
284 def p4_system(cmd):
285     """Specifically invoke p4 as the system command. """
286     real_cmd = p4_build_cmd(cmd)
287     expand = isinstance(real_cmd, basestring)
288     retcode = subprocess.call(real_cmd, shell=expand)
289     if retcode:
290         raise CalledProcessError(retcode, real_cmd)
291
292 def die_bad_access(s):
293     die("failure accessing depot: {0}".format(s.rstrip()))
294
295 def p4_check_access(min_expiration=1):
296     """ Check if we can access Perforce - account still logged in
297     """
298     results = p4CmdList(["login", "-s"])
299
300     if len(results) == 0:
301         # should never get here: always get either some results, or a p4ExitCode
302         assert("could not parse response from perforce")
303
304     result = results[0]
305
306     if 'p4ExitCode' in result:
307         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308         die_bad_access("could not run p4")
309
310     code = result.get("code")
311     if not code:
312         # we get here if we couldn't connect and there was nothing to unmarshal
313         die_bad_access("could not connect")
314
315     elif code == "stat":
316         expiry = result.get("TicketExpiration")
317         if expiry:
318             expiry = int(expiry)
319             if expiry > min_expiration:
320                 # ok to carry on
321                 return
322             else:
323                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325         else:
326             # account without a timeout - all ok
327             return
328
329     elif code == "error":
330         data = result.get("data")
331         if data:
332             die_bad_access("p4 error: {0}".format(data))
333         else:
334             die_bad_access("unknown error")
335     elif code == "info":
336         return
337     else:
338         die_bad_access("unknown error code {0}".format(code))
339
340 _p4_version_string = None
341 def p4_version_string():
342     """Read the version string, showing just the last line, which
343        hopefully is the interesting version bit.
344
345        $ p4 -V
346        Perforce - The Fast Software Configuration Management System.
347        Copyright 1995-2011 Perforce Software.  All rights reserved.
348        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
349     """
350     global _p4_version_string
351     if not _p4_version_string:
352         a = p4_read_pipe_lines(["-V"])
353         _p4_version_string = a[-1].rstrip()
354     return _p4_version_string
355
356 def p4_integrate(src, dest):
357     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
358
359 def p4_sync(f, *options):
360     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
361
362 def p4_add(f):
363     # forcibly add file names with wildcards
364     if wildcard_present(f):
365         p4_system(["add", "-f", f])
366     else:
367         p4_system(["add", f])
368
369 def p4_delete(f):
370     p4_system(["delete", wildcard_encode(f)])
371
372 def p4_edit(f, *options):
373     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
374
375 def p4_revert(f):
376     p4_system(["revert", wildcard_encode(f)])
377
378 def p4_reopen(type, f):
379     p4_system(["reopen", "-t", type, wildcard_encode(f)])
380
381 def p4_reopen_in_change(changelist, files):
382     cmd = ["reopen", "-c", str(changelist)] + files
383     p4_system(cmd)
384
385 def p4_move(src, dest):
386     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
387
388 def p4_last_change():
389     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
390     return int(results[0]['change'])
391
392 def p4_describe(change, shelved=False):
393     """Make sure it returns a valid result by checking for
394        the presence of field "time".  Return a dict of the
395        results."""
396
397     cmd = ["describe", "-s"]
398     if shelved:
399         cmd += ["-S"]
400     cmd += [str(change)]
401
402     ds = p4CmdList(cmd, skip_info=True)
403     if len(ds) != 1:
404         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
405
406     d = ds[0]
407
408     if "p4ExitCode" in d:
409         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
410                                                       str(d)))
411     if "code" in d:
412         if d["code"] == "error":
413             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
414
415     if "time" not in d:
416         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
417
418     return d
419
420 #
421 # Canonicalize the p4 type and return a tuple of the
422 # base type, plus any modifiers.  See "p4 help filetypes"
423 # for a list and explanation.
424 #
425 def split_p4_type(p4type):
426
427     p4_filetypes_historical = {
428         "ctempobj": "binary+Sw",
429         "ctext": "text+C",
430         "cxtext": "text+Cx",
431         "ktext": "text+k",
432         "kxtext": "text+kx",
433         "ltext": "text+F",
434         "tempobj": "binary+FSw",
435         "ubinary": "binary+F",
436         "uresource": "resource+F",
437         "uxbinary": "binary+Fx",
438         "xbinary": "binary+x",
439         "xltext": "text+Fx",
440         "xtempobj": "binary+Swx",
441         "xtext": "text+x",
442         "xunicode": "unicode+x",
443         "xutf16": "utf16+x",
444     }
445     if p4type in p4_filetypes_historical:
446         p4type = p4_filetypes_historical[p4type]
447     mods = ""
448     s = p4type.split("+")
449     base = s[0]
450     mods = ""
451     if len(s) > 1:
452         mods = s[1]
453     return (base, mods)
454
455 #
456 # return the raw p4 type of a file (text, text+ko, etc)
457 #
458 def p4_type(f):
459     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
460     return results[0]['headType']
461
462 #
463 # Given a type base and modifier, return a regexp matching
464 # the keywords that can be expanded in the file
465 #
466 def p4_keywords_regexp_for_type(base, type_mods):
467     if base in ("text", "unicode", "binary"):
468         kwords = None
469         if "ko" in type_mods:
470             kwords = 'Id|Header'
471         elif "k" in type_mods:
472             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
473         else:
474             return None
475         pattern = r"""
476             \$              # Starts with a dollar, followed by...
477             (%s)            # one of the keywords, followed by...
478             (:[^$\n]+)?     # possibly an old expansion, followed by...
479             \$              # another dollar
480             """ % kwords
481         return pattern
482     else:
483         return None
484
485 #
486 # Given a file, return a regexp matching the possible
487 # RCS keywords that will be expanded, or None for files
488 # with kw expansion turned off.
489 #
490 def p4_keywords_regexp_for_file(file):
491     if not os.path.exists(file):
492         return None
493     else:
494         (type_base, type_mods) = split_p4_type(p4_type(file))
495         return p4_keywords_regexp_for_type(type_base, type_mods)
496
497 def setP4ExecBit(file, mode):
498     # Reopens an already open file and changes the execute bit to match
499     # the execute bit setting in the passed in mode.
500
501     p4Type = "+x"
502
503     if not isModeExec(mode):
504         p4Type = getP4OpenedType(file)
505         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
506         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
507         if p4Type[-1] == "+":
508             p4Type = p4Type[0:-1]
509
510     p4_reopen(p4Type, file)
511
512 def getP4OpenedType(file):
513     # Returns the perforce file type for the given file.
514
515     result = p4_read_pipe(["opened", wildcard_encode(file)])
516     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
517     if match:
518         return match.group(1)
519     else:
520         die("Could not determine file type for %s (result: '%s')" % (file, result))
521
522 # Return the set of all p4 labels
523 def getP4Labels(depotPaths):
524     labels = set()
525     if isinstance(depotPaths,basestring):
526         depotPaths = [depotPaths]
527
528     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
529         label = l['label']
530         labels.add(label)
531
532     return labels
533
534 # Return the set of all git tags
535 def getGitTags():
536     gitTags = set()
537     for line in read_pipe_lines(["git", "tag"]):
538         tag = line.strip()
539         gitTags.add(tag)
540     return gitTags
541
542 def diffTreePattern():
543     # This is a simple generator for the diff tree regex pattern. This could be
544     # a class variable if this and parseDiffTreeEntry were a part of a class.
545     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
546     while True:
547         yield pattern
548
549 def parseDiffTreeEntry(entry):
550     """Parses a single diff tree entry into its component elements.
551
552     See git-diff-tree(1) manpage for details about the format of the diff
553     output. This method returns a dictionary with the following elements:
554
555     src_mode - The mode of the source file
556     dst_mode - The mode of the destination file
557     src_sha1 - The sha1 for the source file
558     dst_sha1 - The sha1 fr the destination file
559     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
560     status_score - The score for the status (applicable for 'C' and 'R'
561                    statuses). This is None if there is no score.
562     src - The path for the source file.
563     dst - The path for the destination file. This is only present for
564           copy or renames. If it is not present, this is None.
565
566     If the pattern is not matched, None is returned."""
567
568     match = diffTreePattern().next().match(entry)
569     if match:
570         return {
571             'src_mode': match.group(1),
572             'dst_mode': match.group(2),
573             'src_sha1': match.group(3),
574             'dst_sha1': match.group(4),
575             'status': match.group(5),
576             'status_score': match.group(6),
577             'src': match.group(7),
578             'dst': match.group(10)
579         }
580     return None
581
582 def isModeExec(mode):
583     # Returns True if the given git mode represents an executable file,
584     # otherwise False.
585     return mode[-3:] == "755"
586
587 class P4Exception(Exception):
588     """ Base class for exceptions from the p4 client """
589     def __init__(self, exit_code):
590         self.p4ExitCode = exit_code
591
592 class P4ServerException(P4Exception):
593     """ Base class for exceptions where we get some kind of marshalled up result from the server """
594     def __init__(self, exit_code, p4_result):
595         super(P4ServerException, self).__init__(exit_code)
596         self.p4_result = p4_result
597         self.code = p4_result[0]['code']
598         self.data = p4_result[0]['data']
599
600 class P4RequestSizeException(P4ServerException):
601     """ One of the maxresults or maxscanrows errors """
602     def __init__(self, exit_code, p4_result, limit):
603         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
604         self.limit = limit
605
606 def isModeExecChanged(src_mode, dst_mode):
607     return isModeExec(src_mode) != isModeExec(dst_mode)
608
609 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
610         errors_as_exceptions=False):
611
612     if isinstance(cmd,basestring):
613         cmd = "-G " + cmd
614         expand = True
615     else:
616         cmd = ["-G"] + cmd
617         expand = False
618
619     cmd = p4_build_cmd(cmd)
620     if verbose:
621         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
622
623     # Use a temporary file to avoid deadlocks without
624     # subprocess.communicate(), which would put another copy
625     # of stdout into memory.
626     stdin_file = None
627     if stdin is not None:
628         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
629         if isinstance(stdin,basestring):
630             stdin_file.write(stdin)
631         else:
632             for i in stdin:
633                 stdin_file.write(i + '\n')
634         stdin_file.flush()
635         stdin_file.seek(0)
636
637     p4 = subprocess.Popen(cmd,
638                           shell=expand,
639                           stdin=stdin_file,
640                           stdout=subprocess.PIPE)
641
642     result = []
643     try:
644         while True:
645             entry = marshal.load(p4.stdout)
646             if skip_info:
647                 if 'code' in entry and entry['code'] == 'info':
648                     continue
649             if cb is not None:
650                 cb(entry)
651             else:
652                 result.append(entry)
653     except EOFError:
654         pass
655     exitCode = p4.wait()
656     if exitCode != 0:
657         if errors_as_exceptions:
658             if len(result) > 0:
659                 data = result[0].get('data')
660                 if data:
661                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
662                     if not m:
663                         m = re.search('Request too large \(over (\d+)\)', data)
664
665                     if m:
666                         limit = int(m.group(1))
667                         raise P4RequestSizeException(exitCode, result, limit)
668
669                 raise P4ServerException(exitCode, result)
670             else:
671                 raise P4Exception(exitCode)
672         else:
673             entry = {}
674             entry["p4ExitCode"] = exitCode
675             result.append(entry)
676
677     return result
678
679 def p4Cmd(cmd):
680     list = p4CmdList(cmd)
681     result = {}
682     for entry in list:
683         result.update(entry)
684     return result;
685
686 def p4Where(depotPath):
687     if not depotPath.endswith("/"):
688         depotPath += "/"
689     depotPathLong = depotPath + "..."
690     outputList = p4CmdList(["where", depotPathLong])
691     output = None
692     for entry in outputList:
693         if "depotFile" in entry:
694             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
695             # The base path always ends with "/...".
696             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
697                 output = entry
698                 break
699         elif "data" in entry:
700             data = entry.get("data")
701             space = data.find(" ")
702             if data[:space] == depotPath:
703                 output = entry
704                 break
705     if output == None:
706         return ""
707     if output["code"] == "error":
708         return ""
709     clientPath = ""
710     if "path" in output:
711         clientPath = output.get("path")
712     elif "data" in output:
713         data = output.get("data")
714         lastSpace = data.rfind(" ")
715         clientPath = data[lastSpace + 1:]
716
717     if clientPath.endswith("..."):
718         clientPath = clientPath[:-3]
719     return clientPath
720
721 def currentGitBranch():
722     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
723
724 def isValidGitDir(path):
725     return git_dir(path) != None
726
727 def parseRevision(ref):
728     return read_pipe("git rev-parse %s" % ref).strip()
729
730 def branchExists(ref):
731     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
732                      ignore_error=True)
733     return len(rev) > 0
734
735 def extractLogMessageFromGitCommit(commit):
736     logMessage = ""
737
738     ## fixme: title is first line of commit, not 1st paragraph.
739     foundTitle = False
740     for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
741        if not foundTitle:
742            if len(log) == 1:
743                foundTitle = True
744            continue
745
746        logMessage += log
747     return logMessage
748
749 def extractSettingsGitLog(log):
750     values = {}
751     for line in log.split("\n"):
752         line = line.strip()
753         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
754         if not m:
755             continue
756
757         assignments = m.group(1).split (':')
758         for a in assignments:
759             vals = a.split ('=')
760             key = vals[0].strip()
761             val = ('='.join (vals[1:])).strip()
762             if val.endswith ('\"') and val.startswith('"'):
763                 val = val[1:-1]
764
765             values[key] = val
766
767     paths = values.get("depot-paths")
768     if not paths:
769         paths = values.get("depot-path")
770     if paths:
771         values['depot-paths'] = paths.split(',')
772     return values
773
774 def gitBranchExists(branch):
775     proc = subprocess.Popen(["git", "rev-parse", branch],
776                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
777     return proc.wait() == 0;
778
779 def gitUpdateRef(ref, newvalue):
780     subprocess.check_call(["git", "update-ref", ref, newvalue])
781
782 def gitDeleteRef(ref):
783     subprocess.check_call(["git", "update-ref", "-d", ref])
784
785 _gitConfig = {}
786
787 def gitConfig(key, typeSpecifier=None):
788     if key not in _gitConfig:
789         cmd = [ "git", "config" ]
790         if typeSpecifier:
791             cmd += [ typeSpecifier ]
792         cmd += [ key ]
793         s = read_pipe(cmd, ignore_error=True)
794         _gitConfig[key] = s.strip()
795     return _gitConfig[key]
796
797 def gitConfigBool(key):
798     """Return a bool, using git config --bool.  It is True only if the
799        variable is set to true, and False if set to false or not present
800        in the config."""
801
802     if key not in _gitConfig:
803         _gitConfig[key] = gitConfig(key, '--bool') == "true"
804     return _gitConfig[key]
805
806 def gitConfigInt(key):
807     if key not in _gitConfig:
808         cmd = [ "git", "config", "--int", key ]
809         s = read_pipe(cmd, ignore_error=True)
810         v = s.strip()
811         try:
812             _gitConfig[key] = int(gitConfig(key, '--int'))
813         except ValueError:
814             _gitConfig[key] = None
815     return _gitConfig[key]
816
817 def gitConfigList(key):
818     if key not in _gitConfig:
819         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
820         _gitConfig[key] = s.strip().splitlines()
821         if _gitConfig[key] == ['']:
822             _gitConfig[key] = []
823     return _gitConfig[key]
824
825 def p4BranchesInGit(branchesAreInRemotes=True):
826     """Find all the branches whose names start with "p4/", looking
827        in remotes or heads as specified by the argument.  Return
828        a dictionary of { branch: revision } for each one found.
829        The branch names are the short names, without any
830        "p4/" prefix."""
831
832     branches = {}
833
834     cmdline = "git rev-parse --symbolic "
835     if branchesAreInRemotes:
836         cmdline += "--remotes"
837     else:
838         cmdline += "--branches"
839
840     for line in read_pipe_lines(cmdline):
841         line = line.strip()
842
843         # only import to p4/
844         if not line.startswith('p4/'):
845             continue
846         # special symbolic ref to p4/master
847         if line == "p4/HEAD":
848             continue
849
850         # strip off p4/ prefix
851         branch = line[len("p4/"):]
852
853         branches[branch] = parseRevision(line)
854
855     return branches
856
857 def branch_exists(branch):
858     """Make sure that the given ref name really exists."""
859
860     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
861     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
862     out, _ = p.communicate()
863     if p.returncode:
864         return False
865     # expect exactly one line of output: the branch name
866     return out.rstrip() == branch
867
868 def findUpstreamBranchPoint(head = "HEAD"):
869     branches = p4BranchesInGit()
870     # map from depot-path to branch name
871     branchByDepotPath = {}
872     for branch in branches.keys():
873         tip = branches[branch]
874         log = extractLogMessageFromGitCommit(tip)
875         settings = extractSettingsGitLog(log)
876         if "depot-paths" in settings:
877             paths = ",".join(settings["depot-paths"])
878             branchByDepotPath[paths] = "remotes/p4/" + branch
879
880     settings = None
881     parent = 0
882     while parent < 65535:
883         commit = head + "~%s" % parent
884         log = extractLogMessageFromGitCommit(commit)
885         settings = extractSettingsGitLog(log)
886         if "depot-paths" in settings:
887             paths = ",".join(settings["depot-paths"])
888             if paths in branchByDepotPath:
889                 return [branchByDepotPath[paths], settings]
890
891         parent = parent + 1
892
893     return ["", settings]
894
895 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
896     if not silent:
897         print("Creating/updating branch(es) in %s based on origin branch(es)"
898                % localRefPrefix)
899
900     originPrefix = "origin/p4/"
901
902     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
903         line = line.strip()
904         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
905             continue
906
907         headName = line[len(originPrefix):]
908         remoteHead = localRefPrefix + headName
909         originHead = line
910
911         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
912         if ('depot-paths' not in original
913             or 'change' not in original):
914             continue
915
916         update = False
917         if not gitBranchExists(remoteHead):
918             if verbose:
919                 print("creating %s" % remoteHead)
920             update = True
921         else:
922             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
923             if 'change' in settings:
924                 if settings['depot-paths'] == original['depot-paths']:
925                     originP4Change = int(original['change'])
926                     p4Change = int(settings['change'])
927                     if originP4Change > p4Change:
928                         print("%s (%s) is newer than %s (%s). "
929                                "Updating p4 branch from origin."
930                                % (originHead, originP4Change,
931                                   remoteHead, p4Change))
932                         update = True
933                 else:
934                     print("Ignoring: %s was imported from %s while "
935                            "%s was imported from %s"
936                            % (originHead, ','.join(original['depot-paths']),
937                               remoteHead, ','.join(settings['depot-paths'])))
938
939         if update:
940             system("git update-ref %s %s" % (remoteHead, originHead))
941
942 def originP4BranchesExist():
943         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
944
945
946 def p4ParseNumericChangeRange(parts):
947     changeStart = int(parts[0][1:])
948     if parts[1] == '#head':
949         changeEnd = p4_last_change()
950     else:
951         changeEnd = int(parts[1])
952
953     return (changeStart, changeEnd)
954
955 def chooseBlockSize(blockSize):
956     if blockSize:
957         return blockSize
958     else:
959         return defaultBlockSize
960
961 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
962     assert depotPaths
963
964     # Parse the change range into start and end. Try to find integer
965     # revision ranges as these can be broken up into blocks to avoid
966     # hitting server-side limits (maxrows, maxscanresults). But if
967     # that doesn't work, fall back to using the raw revision specifier
968     # strings, without using block mode.
969
970     if changeRange is None or changeRange == '':
971         changeStart = 1
972         changeEnd = p4_last_change()
973         block_size = chooseBlockSize(requestedBlockSize)
974     else:
975         parts = changeRange.split(',')
976         assert len(parts) == 2
977         try:
978             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
979             block_size = chooseBlockSize(requestedBlockSize)
980         except ValueError:
981             changeStart = parts[0][1:]
982             changeEnd = parts[1]
983             if requestedBlockSize:
984                 die("cannot use --changes-block-size with non-numeric revisions")
985             block_size = None
986
987     changes = set()
988
989     # Retrieve changes a block at a time, to prevent running
990     # into a MaxResults/MaxScanRows error from the server. If
991     # we _do_ hit one of those errors, turn down the block size
992
993     while True:
994         cmd = ['changes']
995
996         if block_size:
997             end = min(changeEnd, changeStart + block_size)
998             revisionRange = "%d,%d" % (changeStart, end)
999         else:
1000             revisionRange = "%s,%s" % (changeStart, changeEnd)
1001
1002         for p in depotPaths:
1003             cmd += ["%s...@%s" % (p, revisionRange)]
1004
1005         # fetch the changes
1006         try:
1007             result = p4CmdList(cmd, errors_as_exceptions=True)
1008         except P4RequestSizeException as e:
1009             if not block_size:
1010                 block_size = e.limit
1011             elif block_size > e.limit:
1012                 block_size = e.limit
1013             else:
1014                 block_size = max(2, block_size // 2)
1015
1016             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1017             continue
1018         except P4Exception as e:
1019             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1020
1021         # Insert changes in chronological order
1022         for entry in reversed(result):
1023             if 'change' not in entry:
1024                 continue
1025             changes.add(int(entry['change']))
1026
1027         if not block_size:
1028             break
1029
1030         if end >= changeEnd:
1031             break
1032
1033         changeStart = end + 1
1034
1035     changes = sorted(changes)
1036     return changes
1037
1038 def p4PathStartsWith(path, prefix):
1039     # This method tries to remedy a potential mixed-case issue:
1040     #
1041     # If UserA adds  //depot/DirA/file1
1042     # and UserB adds //depot/dira/file2
1043     #
1044     # we may or may not have a problem. If you have core.ignorecase=true,
1045     # we treat DirA and dira as the same directory
1046     if gitConfigBool("core.ignorecase"):
1047         return path.lower().startswith(prefix.lower())
1048     return path.startswith(prefix)
1049
1050 def getClientSpec():
1051     """Look at the p4 client spec, create a View() object that contains
1052        all the mappings, and return it."""
1053
1054     specList = p4CmdList("client -o")
1055     if len(specList) != 1:
1056         die('Output from "client -o" is %d lines, expecting 1' %
1057             len(specList))
1058
1059     # dictionary of all client parameters
1060     entry = specList[0]
1061
1062     # the //client/ name
1063     client_name = entry["Client"]
1064
1065     # just the keys that start with "View"
1066     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1067
1068     # hold this new View
1069     view = View(client_name)
1070
1071     # append the lines, in order, to the view
1072     for view_num in range(len(view_keys)):
1073         k = "View%d" % view_num
1074         if k not in view_keys:
1075             die("Expected view key %s missing" % k)
1076         view.append(entry[k])
1077
1078     return view
1079
1080 def getClientRoot():
1081     """Grab the client directory."""
1082
1083     output = p4CmdList("client -o")
1084     if len(output) != 1:
1085         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1086
1087     entry = output[0]
1088     if "Root" not in entry:
1089         die('Client has no "Root"')
1090
1091     return entry["Root"]
1092
1093 #
1094 # P4 wildcards are not allowed in filenames.  P4 complains
1095 # if you simply add them, but you can force it with "-f", in
1096 # which case it translates them into %xx encoding internally.
1097 #
1098 def wildcard_decode(path):
1099     # Search for and fix just these four characters.  Do % last so
1100     # that fixing it does not inadvertently create new %-escapes.
1101     # Cannot have * in a filename in windows; untested as to
1102     # what p4 would do in such a case.
1103     if not platform.system() == "Windows":
1104         path = path.replace("%2A", "*")
1105     path = path.replace("%23", "#") \
1106                .replace("%40", "@") \
1107                .replace("%25", "%")
1108     return path
1109
1110 def wildcard_encode(path):
1111     # do % first to avoid double-encoding the %s introduced here
1112     path = path.replace("%", "%25") \
1113                .replace("*", "%2A") \
1114                .replace("#", "%23") \
1115                .replace("@", "%40")
1116     return path
1117
1118 def wildcard_present(path):
1119     m = re.search("[*#@%]", path)
1120     return m is not None
1121
1122 class LargeFileSystem(object):
1123     """Base class for large file system support."""
1124
1125     def __init__(self, writeToGitStream):
1126         self.largeFiles = set()
1127         self.writeToGitStream = writeToGitStream
1128
1129     def generatePointer(self, cloneDestination, contentFile):
1130         """Return the content of a pointer file that is stored in Git instead of
1131            the actual content."""
1132         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1133
1134     def pushFile(self, localLargeFile):
1135         """Push the actual content which is not stored in the Git repository to
1136            a server."""
1137         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1138
1139     def hasLargeFileExtension(self, relPath):
1140         return reduce(
1141             lambda a, b: a or b,
1142             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1143             False
1144         )
1145
1146     def generateTempFile(self, contents):
1147         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1148         for d in contents:
1149             contentFile.write(d)
1150         contentFile.close()
1151         return contentFile.name
1152
1153     def exceedsLargeFileThreshold(self, relPath, contents):
1154         if gitConfigInt('git-p4.largeFileThreshold'):
1155             contentsSize = sum(len(d) for d in contents)
1156             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1157                 return True
1158         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159             contentsSize = sum(len(d) for d in contents)
1160             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1161                 return False
1162             contentTempFile = self.generateTempFile(contents)
1163             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1164             with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1165                 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1166                 compressedContentsSize = zf.infolist()[0].compress_size
1167             os.remove(contentTempFile)
1168             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169                 return True
1170         return False
1171
1172     def addLargeFile(self, relPath):
1173         self.largeFiles.add(relPath)
1174
1175     def removeLargeFile(self, relPath):
1176         self.largeFiles.remove(relPath)
1177
1178     def isLargeFile(self, relPath):
1179         return relPath in self.largeFiles
1180
1181     def processContent(self, git_mode, relPath, contents):
1182         """Processes the content of git fast import. This method decides if a
1183            file is stored in the large file system and handles all necessary
1184            steps."""
1185         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186             contentTempFile = self.generateTempFile(contents)
1187             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188             if pointer_git_mode:
1189                 git_mode = pointer_git_mode
1190             if localLargeFile:
1191                 # Move temp file to final location in large file system
1192                 largeFileDir = os.path.dirname(localLargeFile)
1193                 if not os.path.isdir(largeFileDir):
1194                     os.makedirs(largeFileDir)
1195                 shutil.move(contentTempFile, localLargeFile)
1196                 self.addLargeFile(relPath)
1197                 if gitConfigBool('git-p4.largeFilePush'):
1198                     self.pushFile(localLargeFile)
1199                 if verbose:
1200                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201         return (git_mode, contents)
1202
1203 class MockLFS(LargeFileSystem):
1204     """Mock large file system for testing."""
1205
1206     def generatePointer(self, contentFile):
1207         """The pointer content is the original content prefixed with "pointer-".
1208            The local filename of the large file storage is derived from the file content.
1209            """
1210         with open(contentFile, 'r') as f:
1211             content = next(f)
1212             gitMode = '100644'
1213             pointerContents = 'pointer-' + content
1214             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215             return (gitMode, pointerContents, localLargeFile)
1216
1217     def pushFile(self, localLargeFile):
1218         """The remote filename of the large file storage is the same as the local
1219            one but in a different directory.
1220            """
1221         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222         if not os.path.exists(remotePath):
1223             os.makedirs(remotePath)
1224         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1225
1226 class GitLFS(LargeFileSystem):
1227     """Git LFS as backend for the git-p4 large file system.
1228        See https://git-lfs.github.com/ for details."""
1229
1230     def __init__(self, *args):
1231         LargeFileSystem.__init__(self, *args)
1232         self.baseGitAttributes = []
1233
1234     def generatePointer(self, contentFile):
1235         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236            mode and content which is stored in the Git repository instead of
1237            the actual content. Return also the new location of the actual
1238            content.
1239            """
1240         if os.path.getsize(contentFile) == 0:
1241             return (None, '', None)
1242
1243         pointerProcess = subprocess.Popen(
1244             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245             stdout=subprocess.PIPE
1246         )
1247         pointerFile = pointerProcess.stdout.read()
1248         if pointerProcess.wait():
1249             os.remove(contentFile)
1250             die('git-lfs pointer command failed. Did you install the extension?')
1251
1252         # Git LFS removed the preamble in the output of the 'pointer' command
1253         # starting from version 1.2.0. Check for the preamble here to support
1254         # earlier versions.
1255         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256         if pointerFile.startswith('Git LFS pointer for'):
1257             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1258
1259         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260         # if someone use external lfs.storage ( not in local repo git )
1261         lfs_path = gitConfig('lfs.storage')
1262         if not lfs_path:
1263             lfs_path = 'lfs'
1264         if not os.path.isabs(lfs_path):
1265             lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1266         localLargeFile = os.path.join(
1267             lfs_path,
1268             'objects', oid[:2], oid[2:4],
1269             oid,
1270         )
1271         # LFS Spec states that pointer files should not have the executable bit set.
1272         gitMode = '100644'
1273         return (gitMode, pointerFile, localLargeFile)
1274
1275     def pushFile(self, localLargeFile):
1276         uploadProcess = subprocess.Popen(
1277             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1278         )
1279         if uploadProcess.wait():
1280             die('git-lfs push command failed. Did you define a remote?')
1281
1282     def generateGitAttributes(self):
1283         return (
1284             self.baseGitAttributes +
1285             [
1286                 '\n',
1287                 '#\n',
1288                 '# Git LFS (see https://git-lfs.github.com/)\n',
1289                 '#\n',
1290             ] +
1291             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1292                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1293             ] +
1294             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1295                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1296             ]
1297         )
1298
1299     def addLargeFile(self, relPath):
1300         LargeFileSystem.addLargeFile(self, relPath)
1301         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1302
1303     def removeLargeFile(self, relPath):
1304         LargeFileSystem.removeLargeFile(self, relPath)
1305         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1306
1307     def processContent(self, git_mode, relPath, contents):
1308         if relPath == '.gitattributes':
1309             self.baseGitAttributes = contents
1310             return (git_mode, self.generateGitAttributes())
1311         else:
1312             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1313
1314 class Command:
1315     delete_actions = ( "delete", "move/delete", "purge" )
1316     add_actions = ( "add", "branch", "move/add" )
1317
1318     def __init__(self):
1319         self.usage = "usage: %prog [options]"
1320         self.needsGit = True
1321         self.verbose = False
1322
1323     # This is required for the "append" update_shelve action
1324     def ensure_value(self, attr, value):
1325         if not hasattr(self, attr) or getattr(self, attr) is None:
1326             setattr(self, attr, value)
1327         return getattr(self, attr)
1328
1329 class P4UserMap:
1330     def __init__(self):
1331         self.userMapFromPerforceServer = False
1332         self.myP4UserId = None
1333
1334     def p4UserId(self):
1335         if self.myP4UserId:
1336             return self.myP4UserId
1337
1338         results = p4CmdList("user -o")
1339         for r in results:
1340             if 'User' in r:
1341                 self.myP4UserId = r['User']
1342                 return r['User']
1343         die("Could not find your p4 user id")
1344
1345     def p4UserIsMe(self, p4User):
1346         # return True if the given p4 user is actually me
1347         me = self.p4UserId()
1348         if not p4User or p4User != me:
1349             return False
1350         else:
1351             return True
1352
1353     def getUserCacheFilename(self):
1354         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1355         return home + "/.gitp4-usercache.txt"
1356
1357     def getUserMapFromPerforceServer(self):
1358         if self.userMapFromPerforceServer:
1359             return
1360         self.users = {}
1361         self.emails = {}
1362
1363         for output in p4CmdList("users"):
1364             if "User" not in output:
1365                 continue
1366             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1367             self.emails[output["Email"]] = output["User"]
1368
1369         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1370         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1371             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1372             if mapUser and len(mapUser[0]) == 3:
1373                 user = mapUser[0][0]
1374                 fullname = mapUser[0][1]
1375                 email = mapUser[0][2]
1376                 self.users[user] = fullname + " <" + email + ">"
1377                 self.emails[email] = user
1378
1379         s = ''
1380         for (key, val) in self.users.items():
1381             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1382
1383         open(self.getUserCacheFilename(), "wb").write(s)
1384         self.userMapFromPerforceServer = True
1385
1386     def loadUserMapFromCache(self):
1387         self.users = {}
1388         self.userMapFromPerforceServer = False
1389         try:
1390             cache = open(self.getUserCacheFilename(), "rb")
1391             lines = cache.readlines()
1392             cache.close()
1393             for line in lines:
1394                 entry = line.strip().split("\t")
1395                 self.users[entry[0]] = entry[1]
1396         except IOError:
1397             self.getUserMapFromPerforceServer()
1398
1399 class P4Debug(Command):
1400     def __init__(self):
1401         Command.__init__(self)
1402         self.options = []
1403         self.description = "A tool to debug the output of p4 -G."
1404         self.needsGit = False
1405
1406     def run(self, args):
1407         j = 0
1408         for output in p4CmdList(args):
1409             print('Element: %d' % j)
1410             j += 1
1411             print(output)
1412         return True
1413
1414 class P4RollBack(Command):
1415     def __init__(self):
1416         Command.__init__(self)
1417         self.options = [
1418             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1419         ]
1420         self.description = "A tool to debug the multi-branch import. Don't use :)"
1421         self.rollbackLocalBranches = False
1422
1423     def run(self, args):
1424         if len(args) != 1:
1425             return False
1426         maxChange = int(args[0])
1427
1428         if "p4ExitCode" in p4Cmd("changes -m 1"):
1429             die("Problems executing p4");
1430
1431         if self.rollbackLocalBranches:
1432             refPrefix = "refs/heads/"
1433             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1434         else:
1435             refPrefix = "refs/remotes/"
1436             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1437
1438         for line in lines:
1439             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1440                 line = line.strip()
1441                 ref = refPrefix + line
1442                 log = extractLogMessageFromGitCommit(ref)
1443                 settings = extractSettingsGitLog(log)
1444
1445                 depotPaths = settings['depot-paths']
1446                 change = settings['change']
1447
1448                 changed = False
1449
1450                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1451                                                            for p in depotPaths]))) == 0:
1452                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1453                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1454                     continue
1455
1456                 while change and int(change) > maxChange:
1457                     changed = True
1458                     if self.verbose:
1459                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1460                     system("git update-ref %s \"%s^\"" % (ref, ref))
1461                     log = extractLogMessageFromGitCommit(ref)
1462                     settings =  extractSettingsGitLog(log)
1463
1464
1465                     depotPaths = settings['depot-paths']
1466                     change = settings['change']
1467
1468                 if changed:
1469                     print("%s rewound to %s" % (ref, change))
1470
1471         return True
1472
1473 class P4Submit(Command, P4UserMap):
1474
1475     conflict_behavior_choices = ("ask", "skip", "quit")
1476
1477     def __init__(self):
1478         Command.__init__(self)
1479         P4UserMap.__init__(self)
1480         self.options = [
1481                 optparse.make_option("--origin", dest="origin"),
1482                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1483                 # preserve the user, requires relevant p4 permissions
1484                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1485                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1486                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1487                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1488                 optparse.make_option("--conflict", dest="conflict_behavior",
1489                                      choices=self.conflict_behavior_choices),
1490                 optparse.make_option("--branch", dest="branch"),
1491                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1492                                      help="Shelve instead of submit. Shelved files are reverted, "
1493                                      "restoring the workspace to the state before the shelve"),
1494                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1495                                      metavar="CHANGELIST",
1496                                      help="update an existing shelved changelist, implies --shelve, "
1497                                            "repeat in-order for multiple shelved changelists"),
1498                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1499                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1500                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1501                                      help="Disable rebase after submit is completed. Can be useful if you "
1502                                      "work from a local git branch that is not master"),
1503                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1504                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1505         ]
1506         self.description = """Submit changes from git to the perforce depot.\n
1507     The `p4-pre-submit` hook is executed if it exists and is executable.
1508     The hook takes no parameters and nothing from standard input. Exiting with
1509     non-zero status from this script prevents `git-p4 submit` from launching.
1510
1511     One usage scenario is to run unit tests in the hook."""
1512
1513         self.usage += " [name of git branch to submit into perforce depot]"
1514         self.origin = ""
1515         self.detectRenames = False
1516         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1517         self.dry_run = False
1518         self.shelve = False
1519         self.update_shelve = list()
1520         self.commit = ""
1521         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1522         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1523         self.prepare_p4_only = False
1524         self.conflict_behavior = None
1525         self.isWindows = (platform.system() == "Windows")
1526         self.exportLabels = False
1527         self.p4HasMoveCommand = p4_has_move_command()
1528         self.branch = None
1529
1530         if gitConfig('git-p4.largeFileSystem'):
1531             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1532
1533     def check(self):
1534         if len(p4CmdList("opened ...")) > 0:
1535             die("You have files opened with perforce! Close them before starting the sync.")
1536
1537     def separate_jobs_from_description(self, message):
1538         """Extract and return a possible Jobs field in the commit
1539            message.  It goes into a separate section in the p4 change
1540            specification.
1541
1542            A jobs line starts with "Jobs:" and looks like a new field
1543            in a form.  Values are white-space separated on the same
1544            line or on following lines that start with a tab.
1545
1546            This does not parse and extract the full git commit message
1547            like a p4 form.  It just sees the Jobs: line as a marker
1548            to pass everything from then on directly into the p4 form,
1549            but outside the description section.
1550
1551            Return a tuple (stripped log message, jobs string)."""
1552
1553         m = re.search(r'^Jobs:', message, re.MULTILINE)
1554         if m is None:
1555             return (message, None)
1556
1557         jobtext = message[m.start():]
1558         stripped_message = message[:m.start()].rstrip()
1559         return (stripped_message, jobtext)
1560
1561     def prepareLogMessage(self, template, message, jobs):
1562         """Edits the template returned from "p4 change -o" to insert
1563            the message in the Description field, and the jobs text in
1564            the Jobs field."""
1565         result = ""
1566
1567         inDescriptionSection = False
1568
1569         for line in template.split("\n"):
1570             if line.startswith("#"):
1571                 result += line + "\n"
1572                 continue
1573
1574             if inDescriptionSection:
1575                 if line.startswith("Files:") or line.startswith("Jobs:"):
1576                     inDescriptionSection = False
1577                     # insert Jobs section
1578                     if jobs:
1579                         result += jobs + "\n"
1580                 else:
1581                     continue
1582             else:
1583                 if line.startswith("Description:"):
1584                     inDescriptionSection = True
1585                     line += "\n"
1586                     for messageLine in message.split("\n"):
1587                         line += "\t" + messageLine + "\n"
1588
1589             result += line + "\n"
1590
1591         return result
1592
1593     def patchRCSKeywords(self, file, pattern):
1594         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1595         (handle, outFileName) = tempfile.mkstemp(dir='.')
1596         try:
1597             outFile = os.fdopen(handle, "w+")
1598             inFile = open(file, "r")
1599             regexp = re.compile(pattern, re.VERBOSE)
1600             for line in inFile.readlines():
1601                 line = regexp.sub(r'$\1$', line)
1602                 outFile.write(line)
1603             inFile.close()
1604             outFile.close()
1605             # Forcibly overwrite the original file
1606             os.unlink(file)
1607             shutil.move(outFileName, file)
1608         except:
1609             # cleanup our temporary file
1610             os.unlink(outFileName)
1611             print("Failed to strip RCS keywords in %s" % file)
1612             raise
1613
1614         print("Patched up RCS keywords in %s" % file)
1615
1616     def p4UserForCommit(self,id):
1617         # Return the tuple (perforce user,git email) for a given git commit id
1618         self.getUserMapFromPerforceServer()
1619         gitEmail = read_pipe(["git", "log", "--max-count=1",
1620                               "--format=%ae", id])
1621         gitEmail = gitEmail.strip()
1622         if gitEmail not in self.emails:
1623             return (None,gitEmail)
1624         else:
1625             return (self.emails[gitEmail],gitEmail)
1626
1627     def checkValidP4Users(self,commits):
1628         # check if any git authors cannot be mapped to p4 users
1629         for id in commits:
1630             (user,email) = self.p4UserForCommit(id)
1631             if not user:
1632                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1633                 if gitConfigBool("git-p4.allowMissingP4Users"):
1634                     print("%s" % msg)
1635                 else:
1636                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1637
1638     def lastP4Changelist(self):
1639         # Get back the last changelist number submitted in this client spec. This
1640         # then gets used to patch up the username in the change. If the same
1641         # client spec is being used by multiple processes then this might go
1642         # wrong.
1643         results = p4CmdList("client -o")        # find the current client
1644         client = None
1645         for r in results:
1646             if 'Client' in r:
1647                 client = r['Client']
1648                 break
1649         if not client:
1650             die("could not get client spec")
1651         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1652         for r in results:
1653             if 'change' in r:
1654                 return r['change']
1655         die("Could not get changelist number for last submit - cannot patch up user details")
1656
1657     def modifyChangelistUser(self, changelist, newUser):
1658         # fixup the user field of a changelist after it has been submitted.
1659         changes = p4CmdList("change -o %s" % changelist)
1660         if len(changes) != 1:
1661             die("Bad output from p4 change modifying %s to user %s" %
1662                 (changelist, newUser))
1663
1664         c = changes[0]
1665         if c['User'] == newUser: return   # nothing to do
1666         c['User'] = newUser
1667         input = marshal.dumps(c)
1668
1669         result = p4CmdList("change -f -i", stdin=input)
1670         for r in result:
1671             if 'code' in r:
1672                 if r['code'] == 'error':
1673                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1674             if 'data' in r:
1675                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1676                 return
1677         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1678
1679     def canChangeChangelists(self):
1680         # check to see if we have p4 admin or super-user permissions, either of
1681         # which are required to modify changelists.
1682         results = p4CmdList(["protects", self.depotPath])
1683         for r in results:
1684             if 'perm' in r:
1685                 if r['perm'] == 'admin':
1686                     return 1
1687                 if r['perm'] == 'super':
1688                     return 1
1689         return 0
1690
1691     def prepareSubmitTemplate(self, changelist=None):
1692         """Run "p4 change -o" to grab a change specification template.
1693            This does not use "p4 -G", as it is nice to keep the submission
1694            template in original order, since a human might edit it.
1695
1696            Remove lines in the Files section that show changes to files
1697            outside the depot path we're committing into."""
1698
1699         [upstream, settings] = findUpstreamBranchPoint()
1700
1701         template = """\
1702 # A Perforce Change Specification.
1703 #
1704 #  Change:      The change number. 'new' on a new changelist.
1705 #  Date:        The date this specification was last modified.
1706 #  Client:      The client on which the changelist was created.  Read-only.
1707 #  User:        The user who created the changelist.
1708 #  Status:      Either 'pending' or 'submitted'. Read-only.
1709 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1710 #  Description: Comments about the changelist.  Required.
1711 #  Jobs:        What opened jobs are to be closed by this changelist.
1712 #               You may delete jobs from this list.  (New changelists only.)
1713 #  Files:       What opened files from the default changelist are to be added
1714 #               to this changelist.  You may delete files from this list.
1715 #               (New changelists only.)
1716 """
1717         files_list = []
1718         inFilesSection = False
1719         change_entry = None
1720         args = ['change', '-o']
1721         if changelist:
1722             args.append(str(changelist))
1723         for entry in p4CmdList(args):
1724             if 'code' not in entry:
1725                 continue
1726             if entry['code'] == 'stat':
1727                 change_entry = entry
1728                 break
1729         if not change_entry:
1730             die('Failed to decode output of p4 change -o')
1731         for key, value in change_entry.iteritems():
1732             if key.startswith('File'):
1733                 if 'depot-paths' in settings:
1734                     if not [p for p in settings['depot-paths']
1735                             if p4PathStartsWith(value, p)]:
1736                         continue
1737                 else:
1738                     if not p4PathStartsWith(value, self.depotPath):
1739                         continue
1740                 files_list.append(value)
1741                 continue
1742         # Output in the order expected by prepareLogMessage
1743         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1744             if key not in change_entry:
1745                 continue
1746             template += '\n'
1747             template += key + ':'
1748             if key == 'Description':
1749                 template += '\n'
1750             for field_line in change_entry[key].splitlines():
1751                 template += '\t'+field_line+'\n'
1752         if len(files_list) > 0:
1753             template += '\n'
1754             template += 'Files:\n'
1755         for path in files_list:
1756             template += '\t'+path+'\n'
1757         return template
1758
1759     def edit_template(self, template_file):
1760         """Invoke the editor to let the user change the submission
1761            message.  Return true if okay to continue with the submit."""
1762
1763         # if configured to skip the editing part, just submit
1764         if gitConfigBool("git-p4.skipSubmitEdit"):
1765             return True
1766
1767         # look at the modification time, to check later if the user saved
1768         # the file
1769         mtime = os.stat(template_file).st_mtime
1770
1771         # invoke the editor
1772         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1773             editor = os.environ.get("P4EDITOR")
1774         else:
1775             editor = read_pipe("git var GIT_EDITOR").strip()
1776         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1777
1778         # If the file was not saved, prompt to see if this patch should
1779         # be skipped.  But skip this verification step if configured so.
1780         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1781             return True
1782
1783         # modification time updated means user saved the file
1784         if os.stat(template_file).st_mtime > mtime:
1785             return True
1786
1787         while True:
1788             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1789             if response == 'y':
1790                 return True
1791             if response == 'n':
1792                 return False
1793
1794     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1795         # diff
1796         if "P4DIFF" in os.environ:
1797             del(os.environ["P4DIFF"])
1798         diff = ""
1799         for editedFile in editedFiles:
1800             diff += p4_read_pipe(['diff', '-du',
1801                                   wildcard_encode(editedFile)])
1802
1803         # new file diff
1804         newdiff = ""
1805         for newFile in filesToAdd:
1806             newdiff += "==== new file ====\n"
1807             newdiff += "--- /dev/null\n"
1808             newdiff += "+++ %s\n" % newFile
1809
1810             is_link = os.path.islink(newFile)
1811             expect_link = newFile in symlinks
1812
1813             if is_link and expect_link:
1814                 newdiff += "+%s\n" % os.readlink(newFile)
1815             else:
1816                 f = open(newFile, "r")
1817                 for line in f.readlines():
1818                     newdiff += "+" + line
1819                 f.close()
1820
1821         return (diff + newdiff).replace('\r\n', '\n')
1822
1823     def applyCommit(self, id):
1824         """Apply one commit, return True if it succeeded."""
1825
1826         print("Applying", read_pipe(["git", "show", "-s",
1827                                      "--format=format:%h %s", id]))
1828
1829         (p4User, gitEmail) = self.p4UserForCommit(id)
1830
1831         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1832         filesToAdd = set()
1833         filesToChangeType = set()
1834         filesToDelete = set()
1835         editedFiles = set()
1836         pureRenameCopy = set()
1837         symlinks = set()
1838         filesToChangeExecBit = {}
1839         all_files = list()
1840
1841         for line in diff:
1842             diff = parseDiffTreeEntry(line)
1843             modifier = diff['status']
1844             path = diff['src']
1845             all_files.append(path)
1846
1847             if modifier == "M":
1848                 p4_edit(path)
1849                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1850                     filesToChangeExecBit[path] = diff['dst_mode']
1851                 editedFiles.add(path)
1852             elif modifier == "A":
1853                 filesToAdd.add(path)
1854                 filesToChangeExecBit[path] = diff['dst_mode']
1855                 if path in filesToDelete:
1856                     filesToDelete.remove(path)
1857
1858                 dst_mode = int(diff['dst_mode'], 8)
1859                 if dst_mode == 0o120000:
1860                     symlinks.add(path)
1861
1862             elif modifier == "D":
1863                 filesToDelete.add(path)
1864                 if path in filesToAdd:
1865                     filesToAdd.remove(path)
1866             elif modifier == "C":
1867                 src, dest = diff['src'], diff['dst']
1868                 all_files.append(dest)
1869                 p4_integrate(src, dest)
1870                 pureRenameCopy.add(dest)
1871                 if diff['src_sha1'] != diff['dst_sha1']:
1872                     p4_edit(dest)
1873                     pureRenameCopy.discard(dest)
1874                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1875                     p4_edit(dest)
1876                     pureRenameCopy.discard(dest)
1877                     filesToChangeExecBit[dest] = diff['dst_mode']
1878                 if self.isWindows:
1879                     # turn off read-only attribute
1880                     os.chmod(dest, stat.S_IWRITE)
1881                 os.unlink(dest)
1882                 editedFiles.add(dest)
1883             elif modifier == "R":
1884                 src, dest = diff['src'], diff['dst']
1885                 all_files.append(dest)
1886                 if self.p4HasMoveCommand:
1887                     p4_edit(src)        # src must be open before move
1888                     p4_move(src, dest)  # opens for (move/delete, move/add)
1889                 else:
1890                     p4_integrate(src, dest)
1891                     if diff['src_sha1'] != diff['dst_sha1']:
1892                         p4_edit(dest)
1893                     else:
1894                         pureRenameCopy.add(dest)
1895                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1896                     if not self.p4HasMoveCommand:
1897                         p4_edit(dest)   # with move: already open, writable
1898                     filesToChangeExecBit[dest] = diff['dst_mode']
1899                 if not self.p4HasMoveCommand:
1900                     if self.isWindows:
1901                         os.chmod(dest, stat.S_IWRITE)
1902                     os.unlink(dest)
1903                     filesToDelete.add(src)
1904                 editedFiles.add(dest)
1905             elif modifier == "T":
1906                 filesToChangeType.add(path)
1907             else:
1908                 die("unknown modifier %s for %s" % (modifier, path))
1909
1910         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1911         patchcmd = diffcmd + " | git apply "
1912         tryPatchCmd = patchcmd + "--check -"
1913         applyPatchCmd = patchcmd + "--check --apply -"
1914         patch_succeeded = True
1915
1916         if os.system(tryPatchCmd) != 0:
1917             fixed_rcs_keywords = False
1918             patch_succeeded = False
1919             print("Unfortunately applying the change failed!")
1920
1921             # Patch failed, maybe it's just RCS keyword woes. Look through
1922             # the patch to see if that's possible.
1923             if gitConfigBool("git-p4.attemptRCSCleanup"):
1924                 file = None
1925                 pattern = None
1926                 kwfiles = {}
1927                 for file in editedFiles | filesToDelete:
1928                     # did this file's delta contain RCS keywords?
1929                     pattern = p4_keywords_regexp_for_file(file)
1930
1931                     if pattern:
1932                         # this file is a possibility...look for RCS keywords.
1933                         regexp = re.compile(pattern, re.VERBOSE)
1934                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1935                             if regexp.search(line):
1936                                 if verbose:
1937                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1938                                 kwfiles[file] = pattern
1939                                 break
1940
1941                 for file in kwfiles:
1942                     if verbose:
1943                         print("zapping %s with %s" % (line,pattern))
1944                     # File is being deleted, so not open in p4.  Must
1945                     # disable the read-only bit on windows.
1946                     if self.isWindows and file not in editedFiles:
1947                         os.chmod(file, stat.S_IWRITE)
1948                     self.patchRCSKeywords(file, kwfiles[file])
1949                     fixed_rcs_keywords = True
1950
1951             if fixed_rcs_keywords:
1952                 print("Retrying the patch with RCS keywords cleaned up")
1953                 if os.system(tryPatchCmd) == 0:
1954                     patch_succeeded = True
1955
1956         if not patch_succeeded:
1957             for f in editedFiles:
1958                 p4_revert(f)
1959             return False
1960
1961         #
1962         # Apply the patch for real, and do add/delete/+x handling.
1963         #
1964         system(applyPatchCmd)
1965
1966         for f in filesToChangeType:
1967             p4_edit(f, "-t", "auto")
1968         for f in filesToAdd:
1969             p4_add(f)
1970         for f in filesToDelete:
1971             p4_revert(f)
1972             p4_delete(f)
1973
1974         # Set/clear executable bits
1975         for f in filesToChangeExecBit.keys():
1976             mode = filesToChangeExecBit[f]
1977             setP4ExecBit(f, mode)
1978
1979         update_shelve = 0
1980         if len(self.update_shelve) > 0:
1981             update_shelve = self.update_shelve.pop(0)
1982             p4_reopen_in_change(update_shelve, all_files)
1983
1984         #
1985         # Build p4 change description, starting with the contents
1986         # of the git commit message.
1987         #
1988         logMessage = extractLogMessageFromGitCommit(id)
1989         logMessage = logMessage.strip()
1990         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1991
1992         template = self.prepareSubmitTemplate(update_shelve)
1993         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1994
1995         if self.preserveUser:
1996            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1997
1998         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1999             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2000             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2001             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2002
2003         separatorLine = "######## everything below this line is just the diff #######\n"
2004         if not self.prepare_p4_only:
2005             submitTemplate += separatorLine
2006             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2007
2008         (handle, fileName) = tempfile.mkstemp()
2009         tmpFile = os.fdopen(handle, "w+b")
2010         if self.isWindows:
2011             submitTemplate = submitTemplate.replace("\n", "\r\n")
2012         tmpFile.write(submitTemplate)
2013         tmpFile.close()
2014
2015         if self.prepare_p4_only:
2016             #
2017             # Leave the p4 tree prepared, and the submit template around
2018             # and let the user decide what to do next
2019             #
2020             print()
2021             print("P4 workspace prepared for submission.")
2022             print("To submit or revert, go to client workspace")
2023             print("  " + self.clientPath)
2024             print()
2025             print("To submit, use \"p4 submit\" to write a new description,")
2026             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2027                   " \"git p4\"." % fileName)
2028             print("You can delete the file \"%s\" when finished." % fileName)
2029
2030             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2031                 print("To preserve change ownership by user %s, you must\n" \
2032                       "do \"p4 change -f <change>\" after submitting and\n" \
2033                       "edit the User field.")
2034             if pureRenameCopy:
2035                 print("After submitting, renamed files must be re-synced.")
2036                 print("Invoke \"p4 sync -f\" on each of these files:")
2037                 for f in pureRenameCopy:
2038                     print("  " + f)
2039
2040             print()
2041             print("To revert the changes, use \"p4 revert ...\", and delete")
2042             print("the submit template file \"%s\"" % fileName)
2043             if filesToAdd:
2044                 print("Since the commit adds new files, they must be deleted:")
2045                 for f in filesToAdd:
2046                     print("  " + f)
2047             print()
2048             return True
2049
2050         #
2051         # Let the user edit the change description, then submit it.
2052         #
2053         submitted = False
2054
2055         try:
2056             if self.edit_template(fileName):
2057                 # read the edited message and submit
2058                 tmpFile = open(fileName, "rb")
2059                 message = tmpFile.read()
2060                 tmpFile.close()
2061                 if self.isWindows:
2062                     message = message.replace("\r\n", "\n")
2063                 submitTemplate = message[:message.index(separatorLine)]
2064
2065                 if update_shelve:
2066                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2067                 elif self.shelve:
2068                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2069                 else:
2070                     p4_write_pipe(['submit', '-i'], submitTemplate)
2071                     # The rename/copy happened by applying a patch that created a
2072                     # new file.  This leaves it writable, which confuses p4.
2073                     for f in pureRenameCopy:
2074                         p4_sync(f, "-f")
2075
2076                 if self.preserveUser:
2077                     if p4User:
2078                         # Get last changelist number. Cannot easily get it from
2079                         # the submit command output as the output is
2080                         # unmarshalled.
2081                         changelist = self.lastP4Changelist()
2082                         self.modifyChangelistUser(changelist, p4User)
2083
2084                 submitted = True
2085
2086         finally:
2087             # skip this patch
2088             if not submitted or self.shelve:
2089                 if self.shelve:
2090                     print ("Reverting shelved files.")
2091                 else:
2092                     print ("Submission cancelled, undoing p4 changes.")
2093                 for f in editedFiles | filesToDelete:
2094                     p4_revert(f)
2095                 for f in filesToAdd:
2096                     p4_revert(f)
2097                     os.remove(f)
2098
2099         os.remove(fileName)
2100         return submitted
2101
2102     # Export git tags as p4 labels. Create a p4 label and then tag
2103     # with that.
2104     def exportGitTags(self, gitTags):
2105         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2106         if len(validLabelRegexp) == 0:
2107             validLabelRegexp = defaultLabelRegexp
2108         m = re.compile(validLabelRegexp)
2109
2110         for name in gitTags:
2111
2112             if not m.match(name):
2113                 if verbose:
2114                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2115                 continue
2116
2117             # Get the p4 commit this corresponds to
2118             logMessage = extractLogMessageFromGitCommit(name)
2119             values = extractSettingsGitLog(logMessage)
2120
2121             if 'change' not in values:
2122                 # a tag pointing to something not sent to p4; ignore
2123                 if verbose:
2124                     print("git tag %s does not give a p4 commit" % name)
2125                 continue
2126             else:
2127                 changelist = values['change']
2128
2129             # Get the tag details.
2130             inHeader = True
2131             isAnnotated = False
2132             body = []
2133             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2134                 l = l.strip()
2135                 if inHeader:
2136                     if re.match(r'tag\s+', l):
2137                         isAnnotated = True
2138                     elif re.match(r'\s*$', l):
2139                         inHeader = False
2140                         continue
2141                 else:
2142                     body.append(l)
2143
2144             if not isAnnotated:
2145                 body = ["lightweight tag imported by git p4\n"]
2146
2147             # Create the label - use the same view as the client spec we are using
2148             clientSpec = getClientSpec()
2149
2150             labelTemplate  = "Label: %s\n" % name
2151             labelTemplate += "Description:\n"
2152             for b in body:
2153                 labelTemplate += "\t" + b + "\n"
2154             labelTemplate += "View:\n"
2155             for depot_side in clientSpec.mappings:
2156                 labelTemplate += "\t%s\n" % depot_side
2157
2158             if self.dry_run:
2159                 print("Would create p4 label %s for tag" % name)
2160             elif self.prepare_p4_only:
2161                 print("Not creating p4 label %s for tag due to option" \
2162                       " --prepare-p4-only" % name)
2163             else:
2164                 p4_write_pipe(["label", "-i"], labelTemplate)
2165
2166                 # Use the label
2167                 p4_system(["tag", "-l", name] +
2168                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2169
2170                 if verbose:
2171                     print("created p4 label for tag %s" % name)
2172
2173     def run(self, args):
2174         if len(args) == 0:
2175             self.master = currentGitBranch()
2176         elif len(args) == 1:
2177             self.master = args[0]
2178             if not branchExists(self.master):
2179                 die("Branch %s does not exist" % self.master)
2180         else:
2181             return False
2182
2183         for i in self.update_shelve:
2184             if i <= 0:
2185                 sys.exit("invalid changelist %d" % i)
2186
2187         if self.master:
2188             allowSubmit = gitConfig("git-p4.allowSubmit")
2189             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2190                 die("%s is not in git-p4.allowSubmit" % self.master)
2191
2192         [upstream, settings] = findUpstreamBranchPoint()
2193         self.depotPath = settings['depot-paths'][0]
2194         if len(self.origin) == 0:
2195             self.origin = upstream
2196
2197         if len(self.update_shelve) > 0:
2198             self.shelve = True
2199
2200         if self.preserveUser:
2201             if not self.canChangeChangelists():
2202                 die("Cannot preserve user names without p4 super-user or admin permissions")
2203
2204         # if not set from the command line, try the config file
2205         if self.conflict_behavior is None:
2206             val = gitConfig("git-p4.conflict")
2207             if val:
2208                 if val not in self.conflict_behavior_choices:
2209                     die("Invalid value '%s' for config git-p4.conflict" % val)
2210             else:
2211                 val = "ask"
2212             self.conflict_behavior = val
2213
2214         if self.verbose:
2215             print("Origin branch is " + self.origin)
2216
2217         if len(self.depotPath) == 0:
2218             print("Internal error: cannot locate perforce depot path from existing branches")
2219             sys.exit(128)
2220
2221         self.useClientSpec = False
2222         if gitConfigBool("git-p4.useclientspec"):
2223             self.useClientSpec = True
2224         if self.useClientSpec:
2225             self.clientSpecDirs = getClientSpec()
2226
2227         # Check for the existence of P4 branches
2228         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2229
2230         if self.useClientSpec and not branchesDetected:
2231             # all files are relative to the client spec
2232             self.clientPath = getClientRoot()
2233         else:
2234             self.clientPath = p4Where(self.depotPath)
2235
2236         if self.clientPath == "":
2237             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2238
2239         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2240         self.oldWorkingDirectory = os.getcwd()
2241
2242         # ensure the clientPath exists
2243         new_client_dir = False
2244         if not os.path.exists(self.clientPath):
2245             new_client_dir = True
2246             os.makedirs(self.clientPath)
2247
2248         chdir(self.clientPath, is_client_path=True)
2249         if self.dry_run:
2250             print("Would synchronize p4 checkout in %s" % self.clientPath)
2251         else:
2252             print("Synchronizing p4 checkout...")
2253             if new_client_dir:
2254                 # old one was destroyed, and maybe nobody told p4
2255                 p4_sync("...", "-f")
2256             else:
2257                 p4_sync("...")
2258         self.check()
2259
2260         commits = []
2261         if self.master:
2262             committish = self.master
2263         else:
2264             committish = 'HEAD'
2265
2266         if self.commit != "":
2267             if self.commit.find("..") != -1:
2268                 limits_ish = self.commit.split("..")
2269                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2270                     commits.append(line.strip())
2271                 commits.reverse()
2272             else:
2273                 commits.append(self.commit)
2274         else:
2275             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2276                 commits.append(line.strip())
2277             commits.reverse()
2278
2279         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2280             self.checkAuthorship = False
2281         else:
2282             self.checkAuthorship = True
2283
2284         if self.preserveUser:
2285             self.checkValidP4Users(commits)
2286
2287         #
2288         # Build up a set of options to be passed to diff when
2289         # submitting each commit to p4.
2290         #
2291         if self.detectRenames:
2292             # command-line -M arg
2293             self.diffOpts = "-M"
2294         else:
2295             # If not explicitly set check the config variable
2296             detectRenames = gitConfig("git-p4.detectRenames")
2297
2298             if detectRenames.lower() == "false" or detectRenames == "":
2299                 self.diffOpts = ""
2300             elif detectRenames.lower() == "true":
2301                 self.diffOpts = "-M"
2302             else:
2303                 self.diffOpts = "-M%s" % detectRenames
2304
2305         # no command-line arg for -C or --find-copies-harder, just
2306         # config variables
2307         detectCopies = gitConfig("git-p4.detectCopies")
2308         if detectCopies.lower() == "false" or detectCopies == "":
2309             pass
2310         elif detectCopies.lower() == "true":
2311             self.diffOpts += " -C"
2312         else:
2313             self.diffOpts += " -C%s" % detectCopies
2314
2315         if gitConfigBool("git-p4.detectCopiesHarder"):
2316             self.diffOpts += " --find-copies-harder"
2317
2318         num_shelves = len(self.update_shelve)
2319         if num_shelves > 0 and num_shelves != len(commits):
2320             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2321                      (len(commits), num_shelves))
2322
2323         hooks_path = gitConfig("core.hooksPath")
2324         if len(hooks_path) <= 0:
2325             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2326
2327         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2328         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2329             sys.exit(1)
2330
2331         #
2332         # Apply the commits, one at a time.  On failure, ask if should
2333         # continue to try the rest of the patches, or quit.
2334         #
2335         if self.dry_run:
2336             print("Would apply")
2337         applied = []
2338         last = len(commits) - 1
2339         for i, commit in enumerate(commits):
2340             if self.dry_run:
2341                 print(" ", read_pipe(["git", "show", "-s",
2342                                       "--format=format:%h %s", commit]))
2343                 ok = True
2344             else:
2345                 ok = self.applyCommit(commit)
2346             if ok:
2347                 applied.append(commit)
2348             else:
2349                 if self.prepare_p4_only and i < last:
2350                     print("Processing only the first commit due to option" \
2351                           " --prepare-p4-only")
2352                     break
2353                 if i < last:
2354                     quit = False
2355                     while True:
2356                         # prompt for what to do, or use the option/variable
2357                         if self.conflict_behavior == "ask":
2358                             print("What do you want to do?")
2359                             response = raw_input("[s]kip this commit but apply"
2360                                                  " the rest, or [q]uit? ")
2361                             if not response:
2362                                 continue
2363                         elif self.conflict_behavior == "skip":
2364                             response = "s"
2365                         elif self.conflict_behavior == "quit":
2366                             response = "q"
2367                         else:
2368                             die("Unknown conflict_behavior '%s'" %
2369                                 self.conflict_behavior)
2370
2371                         if response[0] == "s":
2372                             print("Skipping this commit, but applying the rest")
2373                             break
2374                         if response[0] == "q":
2375                             print("Quitting")
2376                             quit = True
2377                             break
2378                     if quit:
2379                         break
2380
2381         chdir(self.oldWorkingDirectory)
2382         shelved_applied = "shelved" if self.shelve else "applied"
2383         if self.dry_run:
2384             pass
2385         elif self.prepare_p4_only:
2386             pass
2387         elif len(commits) == len(applied):
2388             print("All commits {0}!".format(shelved_applied))
2389
2390             sync = P4Sync()
2391             if self.branch:
2392                 sync.branch = self.branch
2393             if self.disable_p4sync:
2394                 sync.sync_origin_only()
2395             else:
2396                 sync.run([])
2397
2398                 if not self.disable_rebase:
2399                     rebase = P4Rebase()
2400                     rebase.rebase()
2401
2402         else:
2403             if len(applied) == 0:
2404                 print("No commits {0}.".format(shelved_applied))
2405             else:
2406                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2407                 for c in commits:
2408                     if c in applied:
2409                         star = "*"
2410                     else:
2411                         star = " "
2412                     print(star, read_pipe(["git", "show", "-s",
2413                                            "--format=format:%h %s",  c]))
2414                 print("You will have to do 'git p4 sync' and rebase.")
2415
2416         if gitConfigBool("git-p4.exportLabels"):
2417             self.exportLabels = True
2418
2419         if self.exportLabels:
2420             p4Labels = getP4Labels(self.depotPath)
2421             gitTags = getGitTags()
2422
2423             missingGitTags = gitTags - p4Labels
2424             self.exportGitTags(missingGitTags)
2425
2426         # exit with error unless everything applied perfectly
2427         if len(commits) != len(applied):
2428                 sys.exit(1)
2429
2430         return True
2431
2432 class View(object):
2433     """Represent a p4 view ("p4 help views"), and map files in a
2434        repo according to the view."""
2435
2436     def __init__(self, client_name):
2437         self.mappings = []
2438         self.client_prefix = "//%s/" % client_name
2439         # cache results of "p4 where" to lookup client file locations
2440         self.client_spec_path_cache = {}
2441
2442     def append(self, view_line):
2443         """Parse a view line, splitting it into depot and client
2444            sides.  Append to self.mappings, preserving order.  This
2445            is only needed for tag creation."""
2446
2447         # Split the view line into exactly two words.  P4 enforces
2448         # structure on these lines that simplifies this quite a bit.
2449         #
2450         # Either or both words may be double-quoted.
2451         # Single quotes do not matter.
2452         # Double-quote marks cannot occur inside the words.
2453         # A + or - prefix is also inside the quotes.
2454         # There are no quotes unless they contain a space.
2455         # The line is already white-space stripped.
2456         # The two words are separated by a single space.
2457         #
2458         if view_line[0] == '"':
2459             # First word is double quoted.  Find its end.
2460             close_quote_index = view_line.find('"', 1)
2461             if close_quote_index <= 0:
2462                 die("No first-word closing quote found: %s" % view_line)
2463             depot_side = view_line[1:close_quote_index]
2464             # skip closing quote and space
2465             rhs_index = close_quote_index + 1 + 1
2466         else:
2467             space_index = view_line.find(" ")
2468             if space_index <= 0:
2469                 die("No word-splitting space found: %s" % view_line)
2470             depot_side = view_line[0:space_index]
2471             rhs_index = space_index + 1
2472
2473         # prefix + means overlay on previous mapping
2474         if depot_side.startswith("+"):
2475             depot_side = depot_side[1:]
2476
2477         # prefix - means exclude this path, leave out of mappings
2478         exclude = False
2479         if depot_side.startswith("-"):
2480             exclude = True
2481             depot_side = depot_side[1:]
2482
2483         if not exclude:
2484             self.mappings.append(depot_side)
2485
2486     def convert_client_path(self, clientFile):
2487         # chop off //client/ part to make it relative
2488         if not clientFile.startswith(self.client_prefix):
2489             die("No prefix '%s' on clientFile '%s'" %
2490                 (self.client_prefix, clientFile))
2491         return clientFile[len(self.client_prefix):]
2492
2493     def update_client_spec_path_cache(self, files):
2494         """ Caching file paths by "p4 where" batch query """
2495
2496         # List depot file paths exclude that already cached
2497         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2498
2499         if len(fileArgs) == 0:
2500             return  # All files in cache
2501
2502         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2503         for res in where_result:
2504             if "code" in res and res["code"] == "error":
2505                 # assume error is "... file(s) not in client view"
2506                 continue
2507             if "clientFile" not in res:
2508                 die("No clientFile in 'p4 where' output")
2509             if "unmap" in res:
2510                 # it will list all of them, but only one not unmap-ped
2511                 continue
2512             if gitConfigBool("core.ignorecase"):
2513                 res['depotFile'] = res['depotFile'].lower()
2514             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2515
2516         # not found files or unmap files set to ""
2517         for depotFile in fileArgs:
2518             if gitConfigBool("core.ignorecase"):
2519                 depotFile = depotFile.lower()
2520             if depotFile not in self.client_spec_path_cache:
2521                 self.client_spec_path_cache[depotFile] = ""
2522
2523     def map_in_client(self, depot_path):
2524         """Return the relative location in the client where this
2525            depot file should live.  Returns "" if the file should
2526            not be mapped in the client."""
2527
2528         if gitConfigBool("core.ignorecase"):
2529             depot_path = depot_path.lower()
2530
2531         if depot_path in self.client_spec_path_cache:
2532             return self.client_spec_path_cache[depot_path]
2533
2534         die( "Error: %s is not found in client spec path" % depot_path )
2535         return ""
2536
2537 def cloneExcludeCallback(option, opt_str, value, parser):
2538     # prepend "/" because the first "/" was consumed as part of the option itself.
2539     # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2540     parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2541
2542 class P4Sync(Command, P4UserMap):
2543
2544     def __init__(self):
2545         Command.__init__(self)
2546         P4UserMap.__init__(self)
2547         self.options = [
2548                 optparse.make_option("--branch", dest="branch"),
2549                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2550                 optparse.make_option("--changesfile", dest="changesFile"),
2551                 optparse.make_option("--silent", dest="silent", action="store_true"),
2552                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2553                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2554                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2555                                      help="Import into refs/heads/ , not refs/remotes"),
2556                 optparse.make_option("--max-changes", dest="maxChanges",
2557                                      help="Maximum number of changes to import"),
2558                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2559                                      help="Internal block size to use when iteratively calling p4 changes"),
2560                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2561                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2562                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2563                                      help="Only sync files that are included in the Perforce Client Spec"),
2564                 optparse.make_option("-/", dest="cloneExclude",
2565                                      action="callback", callback=cloneExcludeCallback, type="string",
2566                                      help="exclude depot path"),
2567         ]
2568         self.description = """Imports from Perforce into a git repository.\n
2569     example:
2570     //depot/my/project/ -- to import the current head
2571     //depot/my/project/@all -- to import everything
2572     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2573
2574     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2575
2576         self.usage += " //depot/path[@revRange]"
2577         self.silent = False
2578         self.createdBranches = set()
2579         self.committedChanges = set()
2580         self.branch = ""
2581         self.detectBranches = False
2582         self.detectLabels = False
2583         self.importLabels = False
2584         self.changesFile = ""
2585         self.syncWithOrigin = True
2586         self.importIntoRemotes = True
2587         self.maxChanges = ""
2588         self.changes_block_size = None
2589         self.keepRepoPath = False
2590         self.depotPaths = None
2591         self.p4BranchesInGit = []
2592         self.cloneExclude = []
2593         self.useClientSpec = False
2594         self.useClientSpec_from_options = False
2595         self.clientSpecDirs = None
2596         self.tempBranches = []
2597         self.tempBranchLocation = "refs/git-p4-tmp"
2598         self.largeFileSystem = None
2599         self.suppress_meta_comment = False
2600
2601         if gitConfig('git-p4.largeFileSystem'):
2602             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2603             self.largeFileSystem = largeFileSystemConstructor(
2604                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2605             )
2606
2607         if gitConfig("git-p4.syncFromOrigin") == "false":
2608             self.syncWithOrigin = False
2609
2610         self.depotPaths = []
2611         self.changeRange = ""
2612         self.previousDepotPaths = []
2613         self.hasOrigin = False
2614
2615         # map from branch depot path to parent branch
2616         self.knownBranches = {}
2617         self.initialParents = {}
2618
2619         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2620         self.labels = {}
2621
2622     # Force a checkpoint in fast-import and wait for it to finish
2623     def checkpoint(self):
2624         self.gitStream.write("checkpoint\n\n")
2625         self.gitStream.write("progress checkpoint\n\n")
2626         out = self.gitOutput.readline()
2627         if self.verbose:
2628             print("checkpoint finished: " + out)
2629
2630     def isPathWanted(self, path):
2631         for p in self.cloneExclude:
2632             if p.endswith("/"):
2633                 if p4PathStartsWith(path, p):
2634                     return False
2635             # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2636             elif path.lower() == p.lower():
2637                 return False
2638         for p in self.depotPaths:
2639             if p4PathStartsWith(path, p):
2640                 return True
2641         return False
2642
2643     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2644         files = []
2645         fnum = 0
2646         while "depotFile%s" % fnum in commit:
2647             path =  commit["depotFile%s" % fnum]
2648             found = self.isPathWanted(path)
2649             if not found:
2650                 fnum = fnum + 1
2651                 continue
2652
2653             file = {}
2654             file["path"] = path
2655             file["rev"] = commit["rev%s" % fnum]
2656             file["action"] = commit["action%s" % fnum]
2657             file["type"] = commit["type%s" % fnum]
2658             if shelved:
2659                 file["shelved_cl"] = int(shelved_cl)
2660             files.append(file)
2661             fnum = fnum + 1
2662         return files
2663
2664     def extractJobsFromCommit(self, commit):
2665         jobs = []
2666         jnum = 0
2667         while "job%s" % jnum in commit:
2668             job = commit["job%s" % jnum]
2669             jobs.append(job)
2670             jnum = jnum + 1
2671         return jobs
2672
2673     def stripRepoPath(self, path, prefixes):
2674         """When streaming files, this is called to map a p4 depot path
2675            to where it should go in git.  The prefixes are either
2676            self.depotPaths, or self.branchPrefixes in the case of
2677            branch detection."""
2678
2679         if self.useClientSpec:
2680             # branch detection moves files up a level (the branch name)
2681             # from what client spec interpretation gives
2682             path = self.clientSpecDirs.map_in_client(path)
2683             if self.detectBranches:
2684                 for b in self.knownBranches:
2685                     if p4PathStartsWith(path, b + "/"):
2686                         path = path[len(b)+1:]
2687
2688         elif self.keepRepoPath:
2689             # Preserve everything in relative path name except leading
2690             # //depot/; just look at first prefix as they all should
2691             # be in the same depot.
2692             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2693             if p4PathStartsWith(path, depot):
2694                 path = path[len(depot):]
2695
2696         else:
2697             for p in prefixes:
2698                 if p4PathStartsWith(path, p):
2699                     path = path[len(p):]
2700                     break
2701
2702         path = wildcard_decode(path)
2703         return path
2704
2705     def splitFilesIntoBranches(self, commit):
2706         """Look at each depotFile in the commit to figure out to what
2707            branch it belongs."""
2708
2709         if self.clientSpecDirs:
2710             files = self.extractFilesFromCommit(commit)
2711             self.clientSpecDirs.update_client_spec_path_cache(files)
2712
2713         branches = {}
2714         fnum = 0
2715         while "depotFile%s" % fnum in commit:
2716             path =  commit["depotFile%s" % fnum]
2717             found = self.isPathWanted(path)
2718             if not found:
2719                 fnum = fnum + 1
2720                 continue
2721
2722             file = {}
2723             file["path"] = path
2724             file["rev"] = commit["rev%s" % fnum]
2725             file["action"] = commit["action%s" % fnum]
2726             file["type"] = commit["type%s" % fnum]
2727             fnum = fnum + 1
2728
2729             # start with the full relative path where this file would
2730             # go in a p4 client
2731             if self.useClientSpec:
2732                 relPath = self.clientSpecDirs.map_in_client(path)
2733             else:
2734                 relPath = self.stripRepoPath(path, self.depotPaths)
2735
2736             for branch in self.knownBranches.keys():
2737                 # add a trailing slash so that a commit into qt/4.2foo
2738                 # doesn't end up in qt/4.2, e.g.
2739                 if p4PathStartsWith(relPath, branch + "/"):
2740                     if branch not in branches:
2741                         branches[branch] = []
2742                     branches[branch].append(file)
2743                     break
2744
2745         return branches
2746
2747     def writeToGitStream(self, gitMode, relPath, contents):
2748         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2749         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2750         for d in contents:
2751             self.gitStream.write(d)
2752         self.gitStream.write('\n')
2753
2754     def encodeWithUTF8(self, path):
2755         try:
2756             path.decode('ascii')
2757         except:
2758             encoding = 'utf8'
2759             if gitConfig('git-p4.pathEncoding'):
2760                 encoding = gitConfig('git-p4.pathEncoding')
2761             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2762             if self.verbose:
2763                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2764         return path
2765
2766     # output one file from the P4 stream
2767     # - helper for streamP4Files
2768
2769     def streamOneP4File(self, file, contents):
2770         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2771         relPath = self.encodeWithUTF8(relPath)
2772         if verbose:
2773             if 'fileSize' in self.stream_file:
2774                 size = int(self.stream_file['fileSize'])
2775             else:
2776                 size = 0 # deleted files don't get a fileSize apparently
2777             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2778             sys.stdout.flush()
2779
2780         (type_base, type_mods) = split_p4_type(file["type"])
2781
2782         git_mode = "100644"
2783         if "x" in type_mods:
2784             git_mode = "100755"
2785         if type_base == "symlink":
2786             git_mode = "120000"
2787             # p4 print on a symlink sometimes contains "target\n";
2788             # if it does, remove the newline
2789             data = ''.join(contents)
2790             if not data:
2791                 # Some version of p4 allowed creating a symlink that pointed
2792                 # to nothing.  This causes p4 errors when checking out such
2793                 # a change, and errors here too.  Work around it by ignoring
2794                 # the bad symlink; hopefully a future change fixes it.
2795                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2796                 return
2797             elif data[-1] == '\n':
2798                 contents = [data[:-1]]
2799             else:
2800                 contents = [data]
2801
2802         if type_base == "utf16":
2803             # p4 delivers different text in the python output to -G
2804             # than it does when using "print -o", or normal p4 client
2805             # operations.  utf16 is converted to ascii or utf8, perhaps.
2806             # But ascii text saved as -t utf16 is completely mangled.
2807             # Invoke print -o to get the real contents.
2808             #
2809             # On windows, the newlines will always be mangled by print, so put
2810             # them back too.  This is not needed to the cygwin windows version,
2811             # just the native "NT" type.
2812             #
2813             try:
2814                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2815             except Exception as e:
2816                 if 'Translation of file content failed' in str(e):
2817                     type_base = 'binary'
2818                 else:
2819                     raise e
2820             else:
2821                 if p4_version_string().find('/NT') >= 0:
2822                     text = text.replace('\r\n', '\n')
2823                 contents = [ text ]
2824
2825         if type_base == "apple":
2826             # Apple filetype files will be streamed as a concatenation of
2827             # its appledouble header and the contents.  This is useless
2828             # on both macs and non-macs.  If using "print -q -o xx", it
2829             # will create "xx" with the data, and "%xx" with the header.
2830             # This is also not very useful.
2831             #
2832             # Ideally, someday, this script can learn how to generate
2833             # appledouble files directly and import those to git, but
2834             # non-mac machines can never find a use for apple filetype.
2835             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2836             return
2837
2838         # Note that we do not try to de-mangle keywords on utf16 files,
2839         # even though in theory somebody may want that.
2840         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2841         if pattern:
2842             regexp = re.compile(pattern, re.VERBOSE)
2843             text = ''.join(contents)
2844             text = regexp.sub(r'$\1$', text)
2845             contents = [ text ]
2846
2847         if self.largeFileSystem:
2848             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2849
2850         self.writeToGitStream(git_mode, relPath, contents)
2851
2852     def streamOneP4Deletion(self, file):
2853         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2854         relPath = self.encodeWithUTF8(relPath)
2855         if verbose:
2856             sys.stdout.write("delete %s\n" % relPath)
2857             sys.stdout.flush()
2858         self.gitStream.write("D %s\n" % relPath)
2859
2860         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2861             self.largeFileSystem.removeLargeFile(relPath)
2862
2863     # handle another chunk of streaming data
2864     def streamP4FilesCb(self, marshalled):
2865
2866         # catch p4 errors and complain
2867         err = None
2868         if "code" in marshalled:
2869             if marshalled["code"] == "error":
2870                 if "data" in marshalled:
2871                     err = marshalled["data"].rstrip()
2872
2873         if not err and 'fileSize' in self.stream_file:
2874             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2875             if required_bytes > 0:
2876                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2877                     os.getcwd(), required_bytes/1024/1024
2878                 )
2879
2880         if err:
2881             f = None
2882             if self.stream_have_file_info:
2883                 if "depotFile" in self.stream_file:
2884                     f = self.stream_file["depotFile"]
2885             # force a failure in fast-import, else an empty
2886             # commit will be made
2887             self.gitStream.write("\n")
2888             self.gitStream.write("die-now\n")
2889             self.gitStream.close()
2890             # ignore errors, but make sure it exits first
2891             self.importProcess.wait()
2892             if f:
2893                 die("Error from p4 print for %s: %s" % (f, err))
2894             else:
2895                 die("Error from p4 print: %s" % err)
2896
2897         if 'depotFile' in marshalled and self.stream_have_file_info:
2898             # start of a new file - output the old one first
2899             self.streamOneP4File(self.stream_file, self.stream_contents)
2900             self.stream_file = {}
2901             self.stream_contents = []
2902             self.stream_have_file_info = False
2903
2904         # pick up the new file information... for the
2905         # 'data' field we need to append to our array
2906         for k in marshalled.keys():
2907             if k == 'data':
2908                 if 'streamContentSize' not in self.stream_file:
2909                     self.stream_file['streamContentSize'] = 0
2910                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2911                 self.stream_contents.append(marshalled['data'])
2912             else:
2913                 self.stream_file[k] = marshalled[k]
2914
2915         if (verbose and
2916             'streamContentSize' in self.stream_file and
2917             'fileSize' in self.stream_file and
2918             'depotFile' in self.stream_file):
2919             size = int(self.stream_file["fileSize"])
2920             if size > 0:
2921                 progress = 100*self.stream_file['streamContentSize']/size
2922                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2923                 sys.stdout.flush()
2924
2925         self.stream_have_file_info = True
2926
2927     # Stream directly from "p4 files" into "git fast-import"
2928     def streamP4Files(self, files):
2929         filesForCommit = []
2930         filesToRead = []
2931         filesToDelete = []
2932
2933         for f in files:
2934             filesForCommit.append(f)
2935             if f['action'] in self.delete_actions:
2936                 filesToDelete.append(f)
2937             else:
2938                 filesToRead.append(f)
2939
2940         # deleted files...
2941         for f in filesToDelete:
2942             self.streamOneP4Deletion(f)
2943
2944         if len(filesToRead) > 0:
2945             self.stream_file = {}
2946             self.stream_contents = []
2947             self.stream_have_file_info = False
2948
2949             # curry self argument
2950             def streamP4FilesCbSelf(entry):
2951                 self.streamP4FilesCb(entry)
2952
2953             fileArgs = []
2954             for f in filesToRead:
2955                 if 'shelved_cl' in f:
2956                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2957                     # the contents
2958                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2959                 else:
2960                     fileArg = '%s#%s' % (f['path'], f['rev'])
2961
2962                 fileArgs.append(fileArg)
2963
2964             p4CmdList(["-x", "-", "print"],
2965                       stdin=fileArgs,
2966                       cb=streamP4FilesCbSelf)
2967
2968             # do the last chunk
2969             if 'depotFile' in self.stream_file:
2970                 self.streamOneP4File(self.stream_file, self.stream_contents)
2971
2972     def make_email(self, userid):
2973         if userid in self.users:
2974             return self.users[userid]
2975         else:
2976             return "%s <a@b>" % userid
2977
2978     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2979         """ Stream a p4 tag.
2980         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2981         """
2982
2983         if verbose:
2984             print("writing tag %s for commit %s" % (labelName, commit))
2985         gitStream.write("tag %s\n" % labelName)
2986         gitStream.write("from %s\n" % commit)
2987
2988         if 'Owner' in labelDetails:
2989             owner = labelDetails["Owner"]
2990         else:
2991             owner = None
2992
2993         # Try to use the owner of the p4 label, or failing that,
2994         # the current p4 user id.
2995         if owner:
2996             email = self.make_email(owner)
2997         else:
2998             email = self.make_email(self.p4UserId())
2999         tagger = "%s %s %s" % (email, epoch, self.tz)
3000
3001         gitStream.write("tagger %s\n" % tagger)
3002
3003         print("labelDetails=",labelDetails)
3004         if 'Description' in labelDetails:
3005             description = labelDetails['Description']
3006         else:
3007             description = 'Label from git p4'
3008
3009         gitStream.write("data %d\n" % len(description))
3010         gitStream.write(description)
3011         gitStream.write("\n")
3012
3013     def inClientSpec(self, path):
3014         if not self.clientSpecDirs:
3015             return True
3016         inClientSpec = self.clientSpecDirs.map_in_client(path)
3017         if not inClientSpec and self.verbose:
3018             print('Ignoring file outside of client spec: {0}'.format(path))
3019         return inClientSpec
3020
3021     def hasBranchPrefix(self, path):
3022         if not self.branchPrefixes:
3023             return True
3024         hasPrefix = [p for p in self.branchPrefixes
3025                         if p4PathStartsWith(path, p)]
3026         if not hasPrefix and self.verbose:
3027             print('Ignoring file outside of prefix: {0}'.format(path))
3028         return hasPrefix
3029
3030     def commit(self, details, files, branch, parent = "", allow_empty=False):
3031         epoch = details["time"]
3032         author = details["user"]
3033         jobs = self.extractJobsFromCommit(details)
3034
3035         if self.verbose:
3036             print('commit into {0}'.format(branch))
3037
3038         if self.clientSpecDirs:
3039             self.clientSpecDirs.update_client_spec_path_cache(files)
3040
3041         files = [f for f in files
3042             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3043
3044         if gitConfigBool('git-p4.keepEmptyCommits'):
3045             allow_empty = True
3046
3047         if not files and not allow_empty:
3048             print('Ignoring revision {0} as it would produce an empty commit.'
3049                 .format(details['change']))
3050             return
3051
3052         self.gitStream.write("commit %s\n" % branch)
3053         self.gitStream.write("mark :%s\n" % details["change"])
3054         self.committedChanges.add(int(details["change"]))
3055         committer = ""
3056         if author not in self.users:
3057             self.getUserMapFromPerforceServer()
3058         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3059
3060         self.gitStream.write("committer %s\n" % committer)
3061
3062         self.gitStream.write("data <<EOT\n")
3063         self.gitStream.write(details["desc"])
3064         if len(jobs) > 0:
3065             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3066
3067         if not self.suppress_meta_comment:
3068             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3069                                 (','.join(self.branchPrefixes), details["change"]))
3070             if len(details['options']) > 0:
3071                 self.gitStream.write(": options = %s" % details['options'])
3072             self.gitStream.write("]\n")
3073
3074         self.gitStream.write("EOT\n\n")
3075
3076         if len(parent) > 0:
3077             if self.verbose:
3078                 print("parent %s" % parent)
3079             self.gitStream.write("from %s\n" % parent)
3080
3081         self.streamP4Files(files)
3082         self.gitStream.write("\n")
3083
3084         change = int(details["change"])
3085
3086         if change in self.labels:
3087             label = self.labels[change]
3088             labelDetails = label[0]
3089             labelRevisions = label[1]
3090             if self.verbose:
3091                 print("Change %s is labelled %s" % (change, labelDetails))
3092
3093             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3094                                                 for p in self.branchPrefixes])
3095
3096             if len(files) == len(labelRevisions):
3097
3098                 cleanedFiles = {}
3099                 for info in files:
3100                     if info["action"] in self.delete_actions:
3101                         continue
3102                     cleanedFiles[info["depotFile"]] = info["rev"]
3103
3104                 if cleanedFiles == labelRevisions:
3105                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3106
3107                 else:
3108                     if not self.silent:
3109                         print("Tag %s does not match with change %s: files do not match."
3110                                % (labelDetails["label"], change))
3111
3112             else:
3113                 if not self.silent:
3114                     print("Tag %s does not match with change %s: file count is different."
3115                            % (labelDetails["label"], change))
3116
3117     # Build a dictionary of changelists and labels, for "detect-labels" option.
3118     def getLabels(self):
3119         self.labels = {}
3120
3121         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3122         if len(l) > 0 and not self.silent:
3123             print("Finding files belonging to labels in %s" % self.depotPaths)
3124
3125         for output in l:
3126             label = output["label"]
3127             revisions = {}
3128             newestChange = 0
3129             if self.verbose:
3130                 print("Querying files for label %s" % label)
3131             for file in p4CmdList(["files"] +
3132                                       ["%s...@%s" % (p, label)
3133                                           for p in self.depotPaths]):
3134                 revisions[file["depotFile"]] = file["rev"]
3135                 change = int(file["change"])
3136                 if change > newestChange:
3137                     newestChange = change
3138
3139             self.labels[newestChange] = [output, revisions]
3140
3141         if self.verbose:
3142             print("Label changes: %s" % self.labels.keys())
3143
3144     # Import p4 labels as git tags. A direct mapping does not
3145     # exist, so assume that if all the files are at the same revision
3146     # then we can use that, or it's something more complicated we should
3147     # just ignore.
3148     def importP4Labels(self, stream, p4Labels):
3149         if verbose:
3150             print("import p4 labels: " + ' '.join(p4Labels))
3151
3152         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3153         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3154         if len(validLabelRegexp) == 0:
3155             validLabelRegexp = defaultLabelRegexp
3156         m = re.compile(validLabelRegexp)
3157
3158         for name in p4Labels:
3159             commitFound = False
3160
3161             if not m.match(name):
3162                 if verbose:
3163                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3164                 continue
3165
3166             if name in ignoredP4Labels:
3167                 continue
3168
3169             labelDetails = p4CmdList(['label', "-o", name])[0]
3170
3171             # get the most recent changelist for each file in this label
3172             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3173                                 for p in self.depotPaths])
3174
3175             if 'change' in change:
3176                 # find the corresponding git commit; take the oldest commit
3177                 changelist = int(change['change'])
3178                 if changelist in self.committedChanges:
3179                     gitCommit = ":%d" % changelist       # use a fast-import mark
3180                     commitFound = True
3181                 else:
3182                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3183                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3184                     if len(gitCommit) == 0:
3185                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3186                     else:
3187                         commitFound = True
3188                         gitCommit = gitCommit.strip()
3189
3190                 if commitFound:
3191                     # Convert from p4 time format
3192                     try:
3193                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3194                     except ValueError:
3195                         print("Could not convert label time %s" % labelDetails['Update'])
3196                         tmwhen = 1
3197
3198                     when = int(time.mktime(tmwhen))
3199                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3200                     if verbose:
3201                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3202             else:
3203                 if verbose:
3204                     print("Label %s has no changelists - possibly deleted?" % name)
3205
3206             if not commitFound:
3207                 # We can't import this label; don't try again as it will get very
3208                 # expensive repeatedly fetching all the files for labels that will
3209                 # never be imported. If the label is moved in the future, the
3210                 # ignore will need to be removed manually.
3211                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3212
3213     def guessProjectName(self):
3214         for p in self.depotPaths:
3215             if p.endswith("/"):
3216                 p = p[:-1]
3217             p = p[p.strip().rfind("/") + 1:]
3218             if not p.endswith("/"):
3219                p += "/"
3220             return p
3221
3222     def getBranchMapping(self):
3223         lostAndFoundBranches = set()
3224
3225         user = gitConfig("git-p4.branchUser")
3226         if len(user) > 0:
3227             command = "branches -u %s" % user
3228         else:
3229             command = "branches"
3230
3231         for info in p4CmdList(command):
3232             details = p4Cmd(["branch", "-o", info["branch"]])
3233             viewIdx = 0
3234             while "View%s" % viewIdx in details:
3235                 paths = details["View%s" % viewIdx].split(" ")
3236                 viewIdx = viewIdx + 1
3237                 # require standard //depot/foo/... //depot/bar/... mapping
3238                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3239                     continue
3240                 source = paths[0]
3241                 destination = paths[1]
3242                 ## HACK
3243                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3244                     source = source[len(self.depotPaths[0]):-4]
3245                     destination = destination[len(self.depotPaths[0]):-4]
3246
3247                     if destination in self.knownBranches:
3248                         if not self.silent:
3249                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3250                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3251                         continue
3252
3253                     self.knownBranches[destination] = source
3254
3255                     lostAndFoundBranches.discard(destination)
3256
3257                     if source not in self.knownBranches:
3258                         lostAndFoundBranches.add(source)
3259
3260         # Perforce does not strictly require branches to be defined, so we also
3261         # check git config for a branch list.
3262         #
3263         # Example of branch definition in git config file:
3264         # [git-p4]
3265         #   branchList=main:branchA
3266         #   branchList=main:branchB
3267         #   branchList=branchA:branchC
3268         configBranches = gitConfigList("git-p4.branchList")
3269         for branch in configBranches:
3270             if branch:
3271                 (source, destination) = branch.split(":")
3272                 self.knownBranches[destination] = source
3273
3274                 lostAndFoundBranches.discard(destination)
3275
3276                 if source not in self.knownBranches:
3277                     lostAndFoundBranches.add(source)
3278
3279
3280         for branch in lostAndFoundBranches:
3281             self.knownBranches[branch] = branch
3282
3283     def getBranchMappingFromGitBranches(self):
3284         branches = p4BranchesInGit(self.importIntoRemotes)
3285         for branch in branches.keys():
3286             if branch == "master":
3287                 branch = "main"
3288             else:
3289                 branch = branch[len(self.projectName):]
3290             self.knownBranches[branch] = branch
3291
3292     def updateOptionDict(self, d):
3293         option_keys = {}
3294         if self.keepRepoPath:
3295             option_keys['keepRepoPath'] = 1
3296
3297         d["options"] = ' '.join(sorted(option_keys.keys()))
3298
3299     def readOptions(self, d):
3300         self.keepRepoPath = ('options' in d
3301                              and ('keepRepoPath' in d['options']))
3302
3303     def gitRefForBranch(self, branch):
3304         if branch == "main":
3305             return self.refPrefix + "master"
3306
3307         if len(branch) <= 0:
3308             return branch
3309
3310         return self.refPrefix + self.projectName + branch
3311
3312     def gitCommitByP4Change(self, ref, change):
3313         if self.verbose:
3314             print("looking in ref " + ref + " for change %s using bisect..." % change)
3315
3316         earliestCommit = ""
3317         latestCommit = parseRevision(ref)
3318
3319         while True:
3320             if self.verbose:
3321                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3322             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3323             if len(next) == 0:
3324                 if self.verbose:
3325                     print("argh")
3326                 return ""
3327             log = extractLogMessageFromGitCommit(next)
3328             settings = extractSettingsGitLog(log)
3329             currentChange = int(settings['change'])
3330             if self.verbose:
3331                 print("current change %s" % currentChange)
3332
3333             if currentChange == change:
3334                 if self.verbose:
3335                     print("found %s" % next)
3336                 return next
3337
3338             if currentChange < change:
3339                 earliestCommit = "^%s" % next
3340             else:
3341                 if next == latestCommit:
3342                     die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3343                 latestCommit = "%s^@" % next
3344
3345         return ""
3346
3347     def importNewBranch(self, branch, maxChange):
3348         # make fast-import flush all changes to disk and update the refs using the checkpoint
3349         # command so that we can try to find the branch parent in the git history
3350         self.gitStream.write("checkpoint\n\n");
3351         self.gitStream.flush();
3352         branchPrefix = self.depotPaths[0] + branch + "/"
3353         range = "@1,%s" % maxChange
3354         #print "prefix" + branchPrefix
3355         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3356         if len(changes) <= 0:
3357             return False
3358         firstChange = changes[0]
3359         #print "first change in branch: %s" % firstChange
3360         sourceBranch = self.knownBranches[branch]
3361         sourceDepotPath = self.depotPaths[0] + sourceBranch
3362         sourceRef = self.gitRefForBranch(sourceBranch)
3363         #print "source " + sourceBranch
3364
3365         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3366         #print "branch parent: %s" % branchParentChange
3367         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3368         if len(gitParent) > 0:
3369             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3370             #print "parent git commit: %s" % gitParent
3371
3372         self.importChanges(changes)
3373         return True
3374
3375     def searchParent(self, parent, branch, target):
3376         parentFound = False
3377         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3378                                      "--no-merges", parent]):
3379             blob = blob.strip()
3380             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3381                 parentFound = True
3382                 if self.verbose:
3383                     print("Found parent of %s in commit %s" % (branch, blob))
3384                 break
3385         if parentFound:
3386             return blob
3387         else:
3388             return None
3389
3390     def importChanges(self, changes, origin_revision=0):
3391         cnt = 1
3392         for change in changes:
3393             description = p4_describe(change)
3394             self.updateOptionDict(description)
3395
3396             if not self.silent:
3397                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3398                 sys.stdout.flush()
3399             cnt = cnt + 1
3400
3401             try:
3402                 if self.detectBranches:
3403                     branches = self.splitFilesIntoBranches(description)
3404                     for branch in branches.keys():
3405                         ## HACK  --hwn
3406                         branchPrefix = self.depotPaths[0] + branch + "/"
3407                         self.branchPrefixes = [ branchPrefix ]
3408
3409                         parent = ""
3410
3411                         filesForCommit = branches[branch]
3412
3413                         if self.verbose:
3414                             print("branch is %s" % branch)
3415
3416                         self.updatedBranches.add(branch)
3417
3418                         if branch not in self.createdBranches:
3419                             self.createdBranches.add(branch)
3420                             parent = self.knownBranches[branch]
3421                             if parent == branch:
3422                                 parent = ""
3423                             else:
3424                                 fullBranch = self.projectName + branch
3425                                 if fullBranch not in self.p4BranchesInGit:
3426                                     if not self.silent:
3427                                         print("\n    Importing new branch %s" % fullBranch);
3428                                     if self.importNewBranch(branch, change - 1):
3429                                         parent = ""
3430                                         self.p4BranchesInGit.append(fullBranch)
3431                                     if not self.silent:
3432                                         print("\n    Resuming with change %s" % change);
3433
3434                                 if self.verbose:
3435                                     print("parent determined through known branches: %s" % parent)
3436
3437                         branch = self.gitRefForBranch(branch)
3438                         parent = self.gitRefForBranch(parent)
3439
3440                         if self.verbose:
3441                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3442
3443                         if len(parent) == 0 and branch in self.initialParents:
3444                             parent = self.initialParents[branch]
3445                             del self.initialParents[branch]
3446
3447                         blob = None
3448                         if len(parent) > 0:
3449                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3450                             if self.verbose:
3451                                 print("Creating temporary branch: " + tempBranch)
3452                             self.commit(description, filesForCommit, tempBranch)
3453                             self.tempBranches.append(tempBranch)
3454                             self.checkpoint()
3455                             blob = self.searchParent(parent, branch, tempBranch)
3456                         if blob:
3457                             self.commit(description, filesForCommit, branch, blob)
3458                         else:
3459                             if self.verbose:
3460                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3461                             self.commit(description, filesForCommit, branch, parent)
3462                 else:
3463                     files = self.extractFilesFromCommit(description)
3464                     self.commit(description, files, self.branch,
3465                                 self.initialParent)
3466                     # only needed once, to connect to the previous commit
3467                     self.initialParent = ""
3468             except IOError:
3469                 print(self.gitError.read())
3470                 sys.exit(1)
3471
3472     def sync_origin_only(self):
3473         if self.syncWithOrigin:
3474             self.hasOrigin = originP4BranchesExist()
3475             if self.hasOrigin:
3476                 if not self.silent:
3477                     print('Syncing with origin first, using "git fetch origin"')
3478                 system("git fetch origin")
3479
3480     def importHeadRevision(self, revision):
3481         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3482
3483         details = {}
3484         details["user"] = "git perforce import user"
3485         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3486                            % (' '.join(self.depotPaths), revision))
3487         details["change"] = revision
3488         newestRevision = 0
3489
3490         fileCnt = 0
3491         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3492
3493         for info in p4CmdList(["files"] + fileArgs):
3494
3495             if 'code' in info and info['code'] == 'error':
3496                 sys.stderr.write("p4 returned an error: %s\n"
3497                                  % info['data'])
3498                 if info['data'].find("must refer to client") >= 0:
3499                     sys.stderr.write("This particular p4 error is misleading.\n")
3500                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3501                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3502                 sys.exit(1)
3503             if 'p4ExitCode' in info:
3504                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3505                 sys.exit(1)
3506
3507
3508             change = int(info["change"])
3509             if change > newestRevision:
3510                 newestRevision = change
3511
3512             if info["action"] in self.delete_actions:
3513                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3514                 #fileCnt = fileCnt + 1
3515                 continue
3516
3517             for prop in ["depotFile", "rev", "action", "type" ]:
3518                 details["%s%s" % (prop, fileCnt)] = info[prop]
3519
3520             fileCnt = fileCnt + 1
3521
3522         details["change"] = newestRevision
3523
3524         # Use time from top-most change so that all git p4 clones of
3525         # the same p4 repo have the same commit SHA1s.
3526         res = p4_describe(newestRevision)
3527         details["time"] = res["time"]
3528
3529         self.updateOptionDict(details)
3530         try:
3531             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3532         except IOError as err:
3533             print("IO error with git fast-import. Is your git version recent enough?")
3534             print("IO error details: {}".format(err))
3535             print(self.gitError.read())
3536
3537     def openStreams(self):
3538         self.importProcess = subprocess.Popen(["git", "fast-import"],
3539                                               stdin=subprocess.PIPE,
3540                                               stdout=subprocess.PIPE,
3541                                               stderr=subprocess.PIPE);
3542         self.gitOutput = self.importProcess.stdout
3543         self.gitStream = self.importProcess.stdin
3544         self.gitError = self.importProcess.stderr
3545
3546     def closeStreams(self):
3547         self.gitStream.close()
3548         if self.importProcess.wait() != 0:
3549             die("fast-import failed: %s" % self.gitError.read())
3550         self.gitOutput.close()
3551         self.gitError.close()
3552
3553     def run(self, args):
3554         if self.importIntoRemotes:
3555             self.refPrefix = "refs/remotes/p4/"
3556         else:
3557             self.refPrefix = "refs/heads/p4/"
3558
3559         self.sync_origin_only()
3560
3561         branch_arg_given = bool(self.branch)
3562         if len(self.branch) == 0:
3563             self.branch = self.refPrefix + "master"
3564             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3565                 system("git update-ref %s refs/heads/p4" % self.branch)
3566                 system("git branch -D p4")
3567
3568         # accept either the command-line option, or the configuration variable
3569         if self.useClientSpec:
3570             # will use this after clone to set the variable
3571             self.useClientSpec_from_options = True
3572         else:
3573             if gitConfigBool("git-p4.useclientspec"):
3574                 self.useClientSpec = True
3575         if self.useClientSpec:
3576             self.clientSpecDirs = getClientSpec()
3577
3578         # TODO: should always look at previous commits,
3579         # merge with previous imports, if possible.
3580         if args == []:
3581             if self.hasOrigin:
3582                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3583
3584             # branches holds mapping from branch name to sha1
3585             branches = p4BranchesInGit(self.importIntoRemotes)
3586
3587             # restrict to just this one, disabling detect-branches
3588             if branch_arg_given:
3589                 short = self.branch.split("/")[-1]
3590                 if short in branches:
3591                     self.p4BranchesInGit = [ short ]
3592             else:
3593                 self.p4BranchesInGit = branches.keys()
3594
3595             if len(self.p4BranchesInGit) > 1:
3596                 if not self.silent:
3597                     print("Importing from/into multiple branches")
3598                 self.detectBranches = True
3599                 for branch in branches.keys():
3600                     self.initialParents[self.refPrefix + branch] = \
3601                         branches[branch]
3602
3603             if self.verbose:
3604                 print("branches: %s" % self.p4BranchesInGit)
3605
3606             p4Change = 0
3607             for branch in self.p4BranchesInGit:
3608                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3609
3610                 settings = extractSettingsGitLog(logMsg)
3611
3612                 self.readOptions(settings)
3613                 if ('depot-paths' in settings
3614                     and 'change' in settings):
3615                     change = int(settings['change']) + 1
3616                     p4Change = max(p4Change, change)
3617
3618                     depotPaths = sorted(settings['depot-paths'])
3619                     if self.previousDepotPaths == []:
3620                         self.previousDepotPaths = depotPaths
3621                     else:
3622                         paths = []
3623                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3624                             prev_list = prev.split("/")
3625                             cur_list = cur.split("/")
3626                             for i in range(0, min(len(cur_list), len(prev_list))):
3627                                 if cur_list[i] != prev_list[i]:
3628                                     i = i - 1
3629                                     break
3630
3631                             paths.append ("/".join(cur_list[:i + 1]))
3632
3633                         self.previousDepotPaths = paths
3634
3635             if p4Change > 0:
3636                 self.depotPaths = sorted(self.previousDepotPaths)
3637                 self.changeRange = "@%s,#head" % p4Change
3638                 if not self.silent and not self.detectBranches:
3639                     print("Performing incremental import into %s git branch" % self.branch)
3640
3641         # accept multiple ref name abbreviations:
3642         #    refs/foo/bar/branch -> use it exactly
3643         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3644         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3645         if not self.branch.startswith("refs/"):
3646             if self.importIntoRemotes:
3647                 prepend = "refs/remotes/"
3648             else:
3649                 prepend = "refs/heads/"
3650             if not self.branch.startswith("p4/"):
3651                 prepend += "p4/"
3652             self.branch = prepend + self.branch
3653
3654         if len(args) == 0 and self.depotPaths:
3655             if not self.silent:
3656                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3657         else:
3658             if self.depotPaths and self.depotPaths != args:
3659                 print("previous import used depot path %s and now %s was specified. "
3660                        "This doesn't work!" % (' '.join (self.depotPaths),
3661                                                ' '.join (args)))
3662                 sys.exit(1)
3663
3664             self.depotPaths = sorted(args)
3665
3666         revision = ""
3667         self.users = {}
3668
3669         # Make sure no revision specifiers are used when --changesfile
3670         # is specified.
3671         bad_changesfile = False
3672         if len(self.changesFile) > 0:
3673             for p in self.depotPaths:
3674                 if p.find("@") >= 0 or p.find("#") >= 0:
3675                     bad_changesfile = True
3676                     break
3677         if bad_changesfile:
3678             die("Option --changesfile is incompatible with revision specifiers")
3679
3680         newPaths = []
3681         for p in self.depotPaths:
3682             if p.find("@") != -1:
3683                 atIdx = p.index("@")
3684                 self.changeRange = p[atIdx:]
3685                 if self.changeRange == "@all":
3686                     self.changeRange = ""
3687                 elif ',' not in self.changeRange:
3688                     revision = self.changeRange
3689                     self.changeRange = ""
3690                 p = p[:atIdx]
3691             elif p.find("#") != -1:
3692                 hashIdx = p.index("#")
3693                 revision = p[hashIdx:]
3694                 p = p[:hashIdx]
3695             elif self.previousDepotPaths == []:
3696                 # pay attention to changesfile, if given, else import
3697                 # the entire p4 tree at the head revision
3698                 if len(self.changesFile) == 0:
3699                     revision = "#head"
3700
3701             p = re.sub ("\.\.\.$", "", p)
3702             if not p.endswith("/"):
3703                 p += "/"
3704
3705             newPaths.append(p)
3706
3707         self.depotPaths = newPaths
3708
3709         # --detect-branches may change this for each branch
3710         self.branchPrefixes = self.depotPaths
3711
3712         self.loadUserMapFromCache()
3713         self.labels = {}
3714         if self.detectLabels:
3715             self.getLabels();
3716
3717         if self.detectBranches:
3718             ## FIXME - what's a P4 projectName ?
3719             self.projectName = self.guessProjectName()
3720
3721             if self.hasOrigin:
3722                 self.getBranchMappingFromGitBranches()
3723             else:
3724                 self.getBranchMapping()
3725             if self.verbose:
3726                 print("p4-git branches: %s" % self.p4BranchesInGit)
3727                 print("initial parents: %s" % self.initialParents)
3728             for b in self.p4BranchesInGit:
3729                 if b != "master":
3730
3731                     ## FIXME
3732                     b = b[len(self.projectName):]
3733                 self.createdBranches.add(b)
3734
3735         self.openStreams()
3736
3737         if revision:
3738             self.importHeadRevision(revision)
3739         else:
3740             changes = []
3741
3742             if len(self.changesFile) > 0:
3743                 output = open(self.changesFile).readlines()
3744                 changeSet = set()
3745                 for line in output:
3746                     changeSet.add(int(line))
3747
3748                 for change in changeSet:
3749                     changes.append(change)
3750
3751                 changes.sort()
3752             else:
3753                 # catch "git p4 sync" with no new branches, in a repo that
3754                 # does not have any existing p4 branches
3755                 if len(args) == 0:
3756                     if not self.p4BranchesInGit:
3757                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3758
3759                     # The default branch is master, unless --branch is used to
3760                     # specify something else.  Make sure it exists, or complain
3761                     # nicely about how to use --branch.
3762                     if not self.detectBranches:
3763                         if not branch_exists(self.branch):
3764                             if branch_arg_given:
3765                                 die("Error: branch %s does not exist." % self.branch)
3766                             else:
3767                                 die("Error: no branch %s; perhaps specify one with --branch." %
3768                                     self.branch)
3769
3770                 if self.verbose:
3771                     print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3772                                                               self.changeRange))
3773                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3774
3775                 if len(self.maxChanges) > 0:
3776                     changes = changes[:min(int(self.maxChanges), len(changes))]
3777
3778             if len(changes) == 0:
3779                 if not self.silent:
3780                     print("No changes to import!")
3781             else:
3782                 if not self.silent and not self.detectBranches:
3783                     print("Import destination: %s" % self.branch)
3784
3785                 self.updatedBranches = set()
3786
3787                 if not self.detectBranches:
3788                     if args:
3789                         # start a new branch
3790                         self.initialParent = ""
3791                     else:
3792                         # build on a previous revision
3793                         self.initialParent = parseRevision(self.branch)
3794
3795                 self.importChanges(changes)
3796
3797                 if not self.silent:
3798                     print("")
3799                     if len(self.updatedBranches) > 0:
3800                         sys.stdout.write("Updated branches: ")
3801                         for b in self.updatedBranches:
3802                             sys.stdout.write("%s " % b)
3803                         sys.stdout.write("\n")
3804
3805         if gitConfigBool("git-p4.importLabels"):
3806             self.importLabels = True
3807
3808         if self.importLabels:
3809             p4Labels = getP4Labels(self.depotPaths)
3810             gitTags = getGitTags()
3811
3812             missingP4Labels = p4Labels - gitTags
3813             self.importP4Labels(self.gitStream, missingP4Labels)
3814
3815         self.closeStreams()
3816
3817         # Cleanup temporary branches created during import
3818         if self.tempBranches != []:
3819             for branch in self.tempBranches:
3820                 read_pipe("git update-ref -d %s" % branch)
3821             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3822
3823         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3824         # a convenient shortcut refname "p4".
3825         if self.importIntoRemotes:
3826             head_ref = self.refPrefix + "HEAD"
3827             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3828                 system(["git", "symbolic-ref", head_ref, self.branch])
3829
3830         return True
3831
3832 class P4Rebase(Command):
3833     def __init__(self):
3834         Command.__init__(self)
3835         self.options = [
3836                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3837         ]
3838         self.importLabels = False
3839         self.description = ("Fetches the latest revision from perforce and "
3840                             + "rebases the current work (branch) against it")
3841
3842     def run(self, args):
3843         sync = P4Sync()
3844         sync.importLabels = self.importLabels
3845         sync.run([])
3846
3847         return self.rebase()
3848
3849     def rebase(self):
3850         if os.system("git update-index --refresh") != 0:
3851             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3852         if len(read_pipe("git diff-index HEAD --")) > 0:
3853             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3854
3855         [upstream, settings] = findUpstreamBranchPoint()
3856         if len(upstream) == 0:
3857             die("Cannot find upstream branchpoint for rebase")
3858
3859         # the branchpoint may be p4/foo~3, so strip off the parent
3860         upstream = re.sub("~[0-9]+$", "", upstream)
3861
3862         print("Rebasing the current branch onto %s" % upstream)
3863         oldHead = read_pipe("git rev-parse HEAD").strip()
3864         system("git rebase %s" % upstream)
3865         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3866         return True
3867
3868 class P4Clone(P4Sync):
3869     def __init__(self):
3870         P4Sync.__init__(self)
3871         self.description = "Creates a new git repository and imports from Perforce into it"
3872         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3873         self.options += [
3874             optparse.make_option("--destination", dest="cloneDestination",
3875                                  action='store', default=None,
3876                                  help="where to leave result of the clone"),
3877             optparse.make_option("--bare", dest="cloneBare",
3878                                  action="store_true", default=False),
3879         ]
3880         self.cloneDestination = None
3881         self.needsGit = False
3882         self.cloneBare = False
3883
3884     def defaultDestination(self, args):
3885         ## TODO: use common prefix of args?
3886         depotPath = args[0]
3887         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3888         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3889         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3890         depotDir = re.sub(r"/$", "", depotDir)
3891         return os.path.split(depotDir)[1]
3892
3893     def run(self, args):
3894         if len(args) < 1:
3895             return False
3896
3897         if self.keepRepoPath and not self.cloneDestination:
3898             sys.stderr.write("Must specify destination for --keep-path\n")
3899             sys.exit(1)
3900
3901         depotPaths = args
3902
3903         if not self.cloneDestination and len(depotPaths) > 1:
3904             self.cloneDestination = depotPaths[-1]
3905             depotPaths = depotPaths[:-1]
3906
3907         for p in depotPaths:
3908             if not p.startswith("//"):
3909                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3910                 return False
3911
3912         if not self.cloneDestination:
3913             self.cloneDestination = self.defaultDestination(args)
3914
3915         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3916
3917         if not os.path.exists(self.cloneDestination):
3918             os.makedirs(self.cloneDestination)
3919         chdir(self.cloneDestination)
3920
3921         init_cmd = [ "git", "init" ]
3922         if self.cloneBare:
3923             init_cmd.append("--bare")
3924         retcode = subprocess.call(init_cmd)
3925         if retcode:
3926             raise CalledProcessError(retcode, init_cmd)
3927
3928         if not P4Sync.run(self, depotPaths):
3929             return False
3930
3931         # create a master branch and check out a work tree
3932         if gitBranchExists(self.branch):
3933             system([ "git", "branch", "master", self.branch ])
3934             if not self.cloneBare:
3935                 system([ "git", "checkout", "-f" ])
3936         else:
3937             print('Not checking out any branch, use ' \
3938                   '"git checkout -q -b master <branch>"')
3939
3940         # auto-set this variable if invoked with --use-client-spec
3941         if self.useClientSpec_from_options:
3942             system("git config --bool git-p4.useclientspec true")
3943
3944         return True
3945
3946 class P4Unshelve(Command):
3947     def __init__(self):
3948         Command.__init__(self)
3949         self.options = []
3950         self.origin = "HEAD"
3951         self.description = "Unshelve a P4 changelist into a git commit"
3952         self.usage = "usage: %prog [options] changelist"
3953         self.options += [
3954                 optparse.make_option("--origin", dest="origin",
3955                     help="Use this base revision instead of the default (%s)" % self.origin),
3956         ]
3957         self.verbose = False
3958         self.noCommit = False
3959         self.destbranch = "refs/remotes/p4-unshelved"
3960
3961     def renameBranch(self, branch_name):
3962         """ Rename the existing branch to branch_name.N
3963         """
3964
3965         found = True
3966         for i in range(0,1000):
3967             backup_branch_name = "{0}.{1}".format(branch_name, i)
3968             if not gitBranchExists(backup_branch_name):
3969                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3970                 gitDeleteRef(branch_name)
3971                 found = True
3972                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3973                 break
3974
3975         if not found:
3976             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3977
3978     def findLastP4Revision(self, starting_point):
3979         """ Look back from starting_point for the first commit created by git-p4
3980             to find the P4 commit we are based on, and the depot-paths.
3981         """
3982
3983         for parent in (range(65535)):
3984             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3985             settings = extractSettingsGitLog(log)
3986             if 'change' in settings:
3987                 return settings
3988
3989         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3990
3991     def createShelveParent(self, change, branch_name, sync, origin):
3992         """ Create a commit matching the parent of the shelved changelist 'change'
3993         """
3994         parent_description = p4_describe(change, shelved=True)
3995         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
3996         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
3997
3998         parent_files = []
3999         for f in files:
4000             # if it was added in the shelved changelist, it won't exist in the parent
4001             if f['action'] in self.add_actions:
4002                 continue
4003
4004             # if it was deleted in the shelved changelist it must not be deleted
4005             # in the parent - we might even need to create it if the origin branch
4006             # does not have it
4007             if f['action'] in self.delete_actions:
4008                 f['action'] = 'add'
4009
4010             parent_files.append(f)
4011
4012         sync.commit(parent_description, parent_files, branch_name,
4013                 parent=origin, allow_empty=True)
4014         print("created parent commit for {0} based on {1} in {2}".format(
4015             change, self.origin, branch_name))
4016
4017     def run(self, args):
4018         if len(args) != 1:
4019             return False
4020
4021         if not gitBranchExists(self.origin):
4022             sys.exit("origin branch {0} does not exist".format(self.origin))
4023
4024         sync = P4Sync()
4025         changes = args
4026
4027         # only one change at a time
4028         change = changes[0]
4029
4030         # if the target branch already exists, rename it
4031         branch_name = "{0}/{1}".format(self.destbranch, change)
4032         if gitBranchExists(branch_name):
4033             self.renameBranch(branch_name)
4034         sync.branch = branch_name
4035
4036         sync.verbose = self.verbose
4037         sync.suppress_meta_comment = True
4038
4039         settings = self.findLastP4Revision(self.origin)
4040         sync.depotPaths = settings['depot-paths']
4041         sync.branchPrefixes = sync.depotPaths
4042
4043         sync.openStreams()
4044         sync.loadUserMapFromCache()
4045         sync.silent = True
4046
4047         # create a commit for the parent of the shelved changelist
4048         self.createShelveParent(change, branch_name, sync, self.origin)
4049
4050         # create the commit for the shelved changelist itself
4051         description = p4_describe(change, True)
4052         files = sync.extractFilesFromCommit(description, True, change)
4053
4054         sync.commit(description, files, branch_name, "")
4055         sync.closeStreams()
4056
4057         print("unshelved changelist {0} into {1}".format(change, branch_name))
4058
4059         return True
4060
4061 class P4Branches(Command):
4062     def __init__(self):
4063         Command.__init__(self)
4064         self.options = [ ]
4065         self.description = ("Shows the git branches that hold imports and their "
4066                             + "corresponding perforce depot paths")
4067         self.verbose = False
4068
4069     def run(self, args):
4070         if originP4BranchesExist():
4071             createOrUpdateBranchesFromOrigin()
4072
4073         cmdline = "git rev-parse --symbolic "
4074         cmdline += " --remotes"
4075
4076         for line in read_pipe_lines(cmdline):
4077             line = line.strip()
4078
4079             if not line.startswith('p4/') or line == "p4/HEAD":
4080                 continue
4081             branch = line
4082
4083             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4084             settings = extractSettingsGitLog(log)
4085
4086             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4087         return True
4088
4089 class HelpFormatter(optparse.IndentedHelpFormatter):
4090     def __init__(self):
4091         optparse.IndentedHelpFormatter.__init__(self)
4092
4093     def format_description(self, description):
4094         if description:
4095             return description + "\n"
4096         else:
4097             return ""
4098
4099 def printUsage(commands):
4100     print("usage: %s <command> [options]" % sys.argv[0])
4101     print("")
4102     print("valid commands: %s" % ", ".join(commands))
4103     print("")
4104     print("Try %s <command> --help for command specific help." % sys.argv[0])
4105     print("")
4106
4107 commands = {
4108     "debug" : P4Debug,
4109     "submit" : P4Submit,
4110     "commit" : P4Submit,
4111     "sync" : P4Sync,
4112     "rebase" : P4Rebase,
4113     "clone" : P4Clone,
4114     "rollback" : P4RollBack,
4115     "branches" : P4Branches,
4116     "unshelve" : P4Unshelve,
4117 }
4118
4119
4120 def main():
4121     if len(sys.argv[1:]) == 0:
4122         printUsage(commands.keys())
4123         sys.exit(2)
4124
4125     cmdName = sys.argv[1]
4126     try:
4127         klass = commands[cmdName]
4128         cmd = klass()
4129     except KeyError:
4130         print("unknown command %s" % cmdName)
4131         print("")
4132         printUsage(commands.keys())
4133         sys.exit(2)
4134
4135     options = cmd.options
4136     cmd.gitdir = os.environ.get("GIT_DIR", None)
4137
4138     args = sys.argv[2:]
4139
4140     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4141     if cmd.needsGit:
4142         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4143
4144     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4145                                    options,
4146                                    description = cmd.description,
4147                                    formatter = HelpFormatter())
4148
4149     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4150     global verbose
4151     verbose = cmd.verbose
4152     if cmd.needsGit:
4153         if cmd.gitdir == None:
4154             cmd.gitdir = os.path.abspath(".git")
4155             if not isValidGitDir(cmd.gitdir):
4156                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4157                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4158                 if os.path.exists(cmd.gitdir):
4159                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4160                     if len(cdup) > 0:
4161                         chdir(cdup);
4162
4163         if not isValidGitDir(cmd.gitdir):
4164             if isValidGitDir(cmd.gitdir + "/.git"):
4165                 cmd.gitdir += "/.git"
4166             else:
4167                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4168
4169         # so git commands invoked from the P4 workspace will succeed
4170         os.environ["GIT_DIR"] = cmd.gitdir
4171
4172     if not cmd.run(args):
4173         parser.print_help()
4174         sys.exit(2)
4175
4176
4177 if __name__ == '__main__':
4178     main()