Sync with Git 2.22.1
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 # support basestring in python3
31 try:
32     unicode = unicode
33 except NameError:
34     # 'unicode' is undefined, must be Python 3
35     str = str
36     unicode = str
37     bytes = bytes
38     basestring = (str,bytes)
39 else:
40     # 'unicode' exists, must be Python 2
41     str = str
42     unicode = unicode
43     bytes = str
44     basestring = basestring
45
46 try:
47     from subprocess import CalledProcessError
48 except ImportError:
49     # from python2.7:subprocess.py
50     # Exception classes used by this module.
51     class CalledProcessError(Exception):
52         """This exception is raised when a process run by check_call() returns
53         a non-zero exit status.  The exit status will be stored in the
54         returncode attribute."""
55         def __init__(self, returncode, cmd):
56             self.returncode = returncode
57             self.cmd = cmd
58         def __str__(self):
59             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
61 verbose = False
62
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
65
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
68
69 p4_access_checked = False
70
71 def p4_build_cmd(cmd):
72     """Build a suitable p4 command line.
73
74     This consolidates building and returning a p4 command line into one
75     location. It means that hooking into the environment, or other configuration
76     can be done more easily.
77     """
78     real_cmd = ["p4"]
79
80     user = gitConfig("git-p4.user")
81     if len(user) > 0:
82         real_cmd += ["-u",user]
83
84     password = gitConfig("git-p4.password")
85     if len(password) > 0:
86         real_cmd += ["-P", password]
87
88     port = gitConfig("git-p4.port")
89     if len(port) > 0:
90         real_cmd += ["-p", port]
91
92     host = gitConfig("git-p4.host")
93     if len(host) > 0:
94         real_cmd += ["-H", host]
95
96     client = gitConfig("git-p4.client")
97     if len(client) > 0:
98         real_cmd += ["-c", client]
99
100     retries = gitConfigInt("git-p4.retries")
101     if retries is None:
102         # Perform 3 retries by default
103         retries = 3
104     if retries > 0:
105         # Provide a way to not pass this option by setting git-p4.retries to 0
106         real_cmd += ["-r", str(retries)]
107
108     if isinstance(cmd,basestring):
109         real_cmd = ' '.join(real_cmd) + ' ' + cmd
110     else:
111         real_cmd += cmd
112
113     # now check that we can actually talk to the server
114     global p4_access_checked
115     if not p4_access_checked:
116         p4_access_checked = True    # suppress access checks in p4_check_access itself
117         p4_check_access()
118
119     return real_cmd
120
121 def git_dir(path):
122     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123         This won't automatically add ".git" to a directory.
124     """
125     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126     if not d or len(d) == 0:
127         return None
128     else:
129         return d
130
131 def chdir(path, is_client_path=False):
132     """Do chdir to the given path, and set the PWD environment
133        variable for use by P4.  It does not look at getcwd() output.
134        Since we're not using the shell, it is necessary to set the
135        PWD environment variable explicitly.
136
137        Normally, expand the path to force it to be absolute.  This
138        addresses the use of relative path names inside P4 settings,
139        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
140        as given; it looks for .p4config using PWD.
141
142        If is_client_path, the path was handed to us directly by p4,
143        and may be a symbolic link.  Do not call os.getcwd() in this
144        case, because it will cause p4 to think that PWD is not inside
145        the client path.
146        """
147
148     os.chdir(path)
149     if not is_client_path:
150         path = os.getcwd()
151     os.environ['PWD'] = path
152
153 def calcDiskFree():
154     """Return free space in bytes on the disk of the given dirname."""
155     if platform.system() == 'Windows':
156         free_bytes = ctypes.c_ulonglong(0)
157         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158         return free_bytes.value
159     else:
160         st = os.statvfs(os.getcwd())
161         return st.f_bavail * st.f_frsize
162
163 def die(msg):
164     if verbose:
165         raise Exception(msg)
166     else:
167         sys.stderr.write(msg + "\n")
168         sys.exit(1)
169
170 def write_pipe(c, stdin):
171     if verbose:
172         sys.stderr.write('Writing pipe: %s\n' % str(c))
173
174     expand = isinstance(c,basestring)
175     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176     pipe = p.stdin
177     val = pipe.write(stdin)
178     pipe.close()
179     if p.wait():
180         die('Command failed: %s' % str(c))
181
182     return val
183
184 def p4_write_pipe(c, stdin):
185     real_cmd = p4_build_cmd(c)
186     return write_pipe(real_cmd, stdin)
187
188 def read_pipe_full(c):
189     """ Read output from  command. Returns a tuple
190         of the return status, stdout text and stderr
191         text.
192     """
193     if verbose:
194         sys.stderr.write('Reading pipe: %s\n' % str(c))
195
196     expand = isinstance(c,basestring)
197     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198     (out, err) = p.communicate()
199     return (p.returncode, out, err)
200
201 def read_pipe(c, ignore_error=False):
202     """ Read output from  command. Returns the output text on
203         success. On failure, terminates execution, unless
204         ignore_error is True, when it returns an empty string.
205     """
206     (retcode, out, err) = read_pipe_full(c)
207     if retcode != 0:
208         if ignore_error:
209             out = ""
210         else:
211             die('Command failed: %s\nError: %s' % (str(c), err))
212     return out
213
214 def read_pipe_text(c):
215     """ Read output from a command with trailing whitespace stripped.
216         On error, returns None.
217     """
218     (retcode, out, err) = read_pipe_full(c)
219     if retcode != 0:
220         return None
221     else:
222         return out.rstrip()
223
224 def p4_read_pipe(c, ignore_error=False):
225     real_cmd = p4_build_cmd(c)
226     return read_pipe(real_cmd, ignore_error)
227
228 def read_pipe_lines(c):
229     if verbose:
230         sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232     expand = isinstance(c, basestring)
233     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234     pipe = p.stdout
235     val = pipe.readlines()
236     if pipe.close() or p.wait():
237         die('Command failed: %s' % str(c))
238
239     return val
240
241 def p4_read_pipe_lines(c):
242     """Specifically invoke p4 on the command supplied. """
243     real_cmd = p4_build_cmd(c)
244     return read_pipe_lines(real_cmd)
245
246 def p4_has_command(cmd):
247     """Ask p4 for help on this command.  If it returns an error, the
248        command does not exist in this version of p4."""
249     real_cmd = p4_build_cmd(["help", cmd])
250     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251                                    stderr=subprocess.PIPE)
252     p.communicate()
253     return p.returncode == 0
254
255 def p4_has_move_command():
256     """See if the move command exists, that it supports -k, and that
257        it has not been administratively disabled.  The arguments
258        must be correct, but the filenames do not have to exist.  Use
259        ones with wildcards so even if they exist, it will fail."""
260
261     if not p4_has_command("move"):
262         return False
263     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265     (out, err) = p.communicate()
266     # return code will be 1 in either case
267     if err.find("Invalid option") >= 0:
268         return False
269     if err.find("disabled") >= 0:
270         return False
271     # assume it failed because @... was invalid changelist
272     return True
273
274 def system(cmd, ignore_error=False):
275     expand = isinstance(cmd,basestring)
276     if verbose:
277         sys.stderr.write("executing %s\n" % str(cmd))
278     retcode = subprocess.call(cmd, shell=expand)
279     if retcode and not ignore_error:
280         raise CalledProcessError(retcode, cmd)
281
282     return retcode
283
284 def p4_system(cmd):
285     """Specifically invoke p4 as the system command. """
286     real_cmd = p4_build_cmd(cmd)
287     expand = isinstance(real_cmd, basestring)
288     retcode = subprocess.call(real_cmd, shell=expand)
289     if retcode:
290         raise CalledProcessError(retcode, real_cmd)
291
292 def die_bad_access(s):
293     die("failure accessing depot: {0}".format(s.rstrip()))
294
295 def p4_check_access(min_expiration=1):
296     """ Check if we can access Perforce - account still logged in
297     """
298     results = p4CmdList(["login", "-s"])
299
300     if len(results) == 0:
301         # should never get here: always get either some results, or a p4ExitCode
302         assert("could not parse response from perforce")
303
304     result = results[0]
305
306     if 'p4ExitCode' in result:
307         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308         die_bad_access("could not run p4")
309
310     code = result.get("code")
311     if not code:
312         # we get here if we couldn't connect and there was nothing to unmarshal
313         die_bad_access("could not connect")
314
315     elif code == "stat":
316         expiry = result.get("TicketExpiration")
317         if expiry:
318             expiry = int(expiry)
319             if expiry > min_expiration:
320                 # ok to carry on
321                 return
322             else:
323                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325         else:
326             # account without a timeout - all ok
327             return
328
329     elif code == "error":
330         data = result.get("data")
331         if data:
332             die_bad_access("p4 error: {0}".format(data))
333         else:
334             die_bad_access("unknown error")
335     elif code == "info":
336         return
337     else:
338         die_bad_access("unknown error code {0}".format(code))
339
340 _p4_version_string = None
341 def p4_version_string():
342     """Read the version string, showing just the last line, which
343        hopefully is the interesting version bit.
344
345        $ p4 -V
346        Perforce - The Fast Software Configuration Management System.
347        Copyright 1995-2011 Perforce Software.  All rights reserved.
348        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
349     """
350     global _p4_version_string
351     if not _p4_version_string:
352         a = p4_read_pipe_lines(["-V"])
353         _p4_version_string = a[-1].rstrip()
354     return _p4_version_string
355
356 def p4_integrate(src, dest):
357     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
358
359 def p4_sync(f, *options):
360     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
361
362 def p4_add(f):
363     # forcibly add file names with wildcards
364     if wildcard_present(f):
365         p4_system(["add", "-f", f])
366     else:
367         p4_system(["add", f])
368
369 def p4_delete(f):
370     p4_system(["delete", wildcard_encode(f)])
371
372 def p4_edit(f, *options):
373     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
374
375 def p4_revert(f):
376     p4_system(["revert", wildcard_encode(f)])
377
378 def p4_reopen(type, f):
379     p4_system(["reopen", "-t", type, wildcard_encode(f)])
380
381 def p4_reopen_in_change(changelist, files):
382     cmd = ["reopen", "-c", str(changelist)] + files
383     p4_system(cmd)
384
385 def p4_move(src, dest):
386     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
387
388 def p4_last_change():
389     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
390     return int(results[0]['change'])
391
392 def p4_describe(change, shelved=False):
393     """Make sure it returns a valid result by checking for
394        the presence of field "time".  Return a dict of the
395        results."""
396
397     cmd = ["describe", "-s"]
398     if shelved:
399         cmd += ["-S"]
400     cmd += [str(change)]
401
402     ds = p4CmdList(cmd, skip_info=True)
403     if len(ds) != 1:
404         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
405
406     d = ds[0]
407
408     if "p4ExitCode" in d:
409         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
410                                                       str(d)))
411     if "code" in d:
412         if d["code"] == "error":
413             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
414
415     if "time" not in d:
416         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
417
418     return d
419
420 #
421 # Canonicalize the p4 type and return a tuple of the
422 # base type, plus any modifiers.  See "p4 help filetypes"
423 # for a list and explanation.
424 #
425 def split_p4_type(p4type):
426
427     p4_filetypes_historical = {
428         "ctempobj": "binary+Sw",
429         "ctext": "text+C",
430         "cxtext": "text+Cx",
431         "ktext": "text+k",
432         "kxtext": "text+kx",
433         "ltext": "text+F",
434         "tempobj": "binary+FSw",
435         "ubinary": "binary+F",
436         "uresource": "resource+F",
437         "uxbinary": "binary+Fx",
438         "xbinary": "binary+x",
439         "xltext": "text+Fx",
440         "xtempobj": "binary+Swx",
441         "xtext": "text+x",
442         "xunicode": "unicode+x",
443         "xutf16": "utf16+x",
444     }
445     if p4type in p4_filetypes_historical:
446         p4type = p4_filetypes_historical[p4type]
447     mods = ""
448     s = p4type.split("+")
449     base = s[0]
450     mods = ""
451     if len(s) > 1:
452         mods = s[1]
453     return (base, mods)
454
455 #
456 # return the raw p4 type of a file (text, text+ko, etc)
457 #
458 def p4_type(f):
459     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
460     return results[0]['headType']
461
462 #
463 # Given a type base and modifier, return a regexp matching
464 # the keywords that can be expanded in the file
465 #
466 def p4_keywords_regexp_for_type(base, type_mods):
467     if base in ("text", "unicode", "binary"):
468         kwords = None
469         if "ko" in type_mods:
470             kwords = 'Id|Header'
471         elif "k" in type_mods:
472             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
473         else:
474             return None
475         pattern = r"""
476             \$              # Starts with a dollar, followed by...
477             (%s)            # one of the keywords, followed by...
478             (:[^$\n]+)?     # possibly an old expansion, followed by...
479             \$              # another dollar
480             """ % kwords
481         return pattern
482     else:
483         return None
484
485 #
486 # Given a file, return a regexp matching the possible
487 # RCS keywords that will be expanded, or None for files
488 # with kw expansion turned off.
489 #
490 def p4_keywords_regexp_for_file(file):
491     if not os.path.exists(file):
492         return None
493     else:
494         (type_base, type_mods) = split_p4_type(p4_type(file))
495         return p4_keywords_regexp_for_type(type_base, type_mods)
496
497 def setP4ExecBit(file, mode):
498     # Reopens an already open file and changes the execute bit to match
499     # the execute bit setting in the passed in mode.
500
501     p4Type = "+x"
502
503     if not isModeExec(mode):
504         p4Type = getP4OpenedType(file)
505         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
506         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
507         if p4Type[-1] == "+":
508             p4Type = p4Type[0:-1]
509
510     p4_reopen(p4Type, file)
511
512 def getP4OpenedType(file):
513     # Returns the perforce file type for the given file.
514
515     result = p4_read_pipe(["opened", wildcard_encode(file)])
516     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
517     if match:
518         return match.group(1)
519     else:
520         die("Could not determine file type for %s (result: '%s')" % (file, result))
521
522 # Return the set of all p4 labels
523 def getP4Labels(depotPaths):
524     labels = set()
525     if isinstance(depotPaths,basestring):
526         depotPaths = [depotPaths]
527
528     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
529         label = l['label']
530         labels.add(label)
531
532     return labels
533
534 # Return the set of all git tags
535 def getGitTags():
536     gitTags = set()
537     for line in read_pipe_lines(["git", "tag"]):
538         tag = line.strip()
539         gitTags.add(tag)
540     return gitTags
541
542 def diffTreePattern():
543     # This is a simple generator for the diff tree regex pattern. This could be
544     # a class variable if this and parseDiffTreeEntry were a part of a class.
545     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
546     while True:
547         yield pattern
548
549 def parseDiffTreeEntry(entry):
550     """Parses a single diff tree entry into its component elements.
551
552     See git-diff-tree(1) manpage for details about the format of the diff
553     output. This method returns a dictionary with the following elements:
554
555     src_mode - The mode of the source file
556     dst_mode - The mode of the destination file
557     src_sha1 - The sha1 for the source file
558     dst_sha1 - The sha1 fr the destination file
559     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
560     status_score - The score for the status (applicable for 'C' and 'R'
561                    statuses). This is None if there is no score.
562     src - The path for the source file.
563     dst - The path for the destination file. This is only present for
564           copy or renames. If it is not present, this is None.
565
566     If the pattern is not matched, None is returned."""
567
568     match = diffTreePattern().next().match(entry)
569     if match:
570         return {
571             'src_mode': match.group(1),
572             'dst_mode': match.group(2),
573             'src_sha1': match.group(3),
574             'dst_sha1': match.group(4),
575             'status': match.group(5),
576             'status_score': match.group(6),
577             'src': match.group(7),
578             'dst': match.group(10)
579         }
580     return None
581
582 def isModeExec(mode):
583     # Returns True if the given git mode represents an executable file,
584     # otherwise False.
585     return mode[-3:] == "755"
586
587 class P4Exception(Exception):
588     """ Base class for exceptions from the p4 client """
589     def __init__(self, exit_code):
590         self.p4ExitCode = exit_code
591
592 class P4ServerException(P4Exception):
593     """ Base class for exceptions where we get some kind of marshalled up result from the server """
594     def __init__(self, exit_code, p4_result):
595         super(P4ServerException, self).__init__(exit_code)
596         self.p4_result = p4_result
597         self.code = p4_result[0]['code']
598         self.data = p4_result[0]['data']
599
600 class P4RequestSizeException(P4ServerException):
601     """ One of the maxresults or maxscanrows errors """
602     def __init__(self, exit_code, p4_result, limit):
603         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
604         self.limit = limit
605
606 def isModeExecChanged(src_mode, dst_mode):
607     return isModeExec(src_mode) != isModeExec(dst_mode)
608
609 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
610         errors_as_exceptions=False):
611
612     if isinstance(cmd,basestring):
613         cmd = "-G " + cmd
614         expand = True
615     else:
616         cmd = ["-G"] + cmd
617         expand = False
618
619     cmd = p4_build_cmd(cmd)
620     if verbose:
621         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
622
623     # Use a temporary file to avoid deadlocks without
624     # subprocess.communicate(), which would put another copy
625     # of stdout into memory.
626     stdin_file = None
627     if stdin is not None:
628         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
629         if isinstance(stdin,basestring):
630             stdin_file.write(stdin)
631         else:
632             for i in stdin:
633                 stdin_file.write(i + '\n')
634         stdin_file.flush()
635         stdin_file.seek(0)
636
637     p4 = subprocess.Popen(cmd,
638                           shell=expand,
639                           stdin=stdin_file,
640                           stdout=subprocess.PIPE)
641
642     result = []
643     try:
644         while True:
645             entry = marshal.load(p4.stdout)
646             if skip_info:
647                 if 'code' in entry and entry['code'] == 'info':
648                     continue
649             if cb is not None:
650                 cb(entry)
651             else:
652                 result.append(entry)
653     except EOFError:
654         pass
655     exitCode = p4.wait()
656     if exitCode != 0:
657         if errors_as_exceptions:
658             if len(result) > 0:
659                 data = result[0].get('data')
660                 if data:
661                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
662                     if not m:
663                         m = re.search('Request too large \(over (\d+)\)', data)
664
665                     if m:
666                         limit = int(m.group(1))
667                         raise P4RequestSizeException(exitCode, result, limit)
668
669                 raise P4ServerException(exitCode, result)
670             else:
671                 raise P4Exception(exitCode)
672         else:
673             entry = {}
674             entry["p4ExitCode"] = exitCode
675             result.append(entry)
676
677     return result
678
679 def p4Cmd(cmd):
680     list = p4CmdList(cmd)
681     result = {}
682     for entry in list:
683         result.update(entry)
684     return result;
685
686 def p4Where(depotPath):
687     if not depotPath.endswith("/"):
688         depotPath += "/"
689     depotPathLong = depotPath + "..."
690     outputList = p4CmdList(["where", depotPathLong])
691     output = None
692     for entry in outputList:
693         if "depotFile" in entry:
694             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
695             # The base path always ends with "/...".
696             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
697                 output = entry
698                 break
699         elif "data" in entry:
700             data = entry.get("data")
701             space = data.find(" ")
702             if data[:space] == depotPath:
703                 output = entry
704                 break
705     if output == None:
706         return ""
707     if output["code"] == "error":
708         return ""
709     clientPath = ""
710     if "path" in output:
711         clientPath = output.get("path")
712     elif "data" in output:
713         data = output.get("data")
714         lastSpace = data.rfind(" ")
715         clientPath = data[lastSpace + 1:]
716
717     if clientPath.endswith("..."):
718         clientPath = clientPath[:-3]
719     return clientPath
720
721 def currentGitBranch():
722     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
723
724 def isValidGitDir(path):
725     return git_dir(path) != None
726
727 def parseRevision(ref):
728     return read_pipe("git rev-parse %s" % ref).strip()
729
730 def branchExists(ref):
731     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
732                      ignore_error=True)
733     return len(rev) > 0
734
735 def extractLogMessageFromGitCommit(commit):
736     logMessage = ""
737
738     ## fixme: title is first line of commit, not 1st paragraph.
739     foundTitle = False
740     for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
741        if not foundTitle:
742            if len(log) == 1:
743                foundTitle = True
744            continue
745
746        logMessage += log
747     return logMessage
748
749 def extractSettingsGitLog(log):
750     values = {}
751     for line in log.split("\n"):
752         line = line.strip()
753         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
754         if not m:
755             continue
756
757         assignments = m.group(1).split (':')
758         for a in assignments:
759             vals = a.split ('=')
760             key = vals[0].strip()
761             val = ('='.join (vals[1:])).strip()
762             if val.endswith ('\"') and val.startswith('"'):
763                 val = val[1:-1]
764
765             values[key] = val
766
767     paths = values.get("depot-paths")
768     if not paths:
769         paths = values.get("depot-path")
770     if paths:
771         values['depot-paths'] = paths.split(',')
772     return values
773
774 def gitBranchExists(branch):
775     proc = subprocess.Popen(["git", "rev-parse", branch],
776                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
777     return proc.wait() == 0;
778
779 def gitUpdateRef(ref, newvalue):
780     subprocess.check_call(["git", "update-ref", ref, newvalue])
781
782 def gitDeleteRef(ref):
783     subprocess.check_call(["git", "update-ref", "-d", ref])
784
785 _gitConfig = {}
786
787 def gitConfig(key, typeSpecifier=None):
788     if key not in _gitConfig:
789         cmd = [ "git", "config" ]
790         if typeSpecifier:
791             cmd += [ typeSpecifier ]
792         cmd += [ key ]
793         s = read_pipe(cmd, ignore_error=True)
794         _gitConfig[key] = s.strip()
795     return _gitConfig[key]
796
797 def gitConfigBool(key):
798     """Return a bool, using git config --bool.  It is True only if the
799        variable is set to true, and False if set to false or not present
800        in the config."""
801
802     if key not in _gitConfig:
803         _gitConfig[key] = gitConfig(key, '--bool') == "true"
804     return _gitConfig[key]
805
806 def gitConfigInt(key):
807     if key not in _gitConfig:
808         cmd = [ "git", "config", "--int", key ]
809         s = read_pipe(cmd, ignore_error=True)
810         v = s.strip()
811         try:
812             _gitConfig[key] = int(gitConfig(key, '--int'))
813         except ValueError:
814             _gitConfig[key] = None
815     return _gitConfig[key]
816
817 def gitConfigList(key):
818     if key not in _gitConfig:
819         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
820         _gitConfig[key] = s.strip().splitlines()
821         if _gitConfig[key] == ['']:
822             _gitConfig[key] = []
823     return _gitConfig[key]
824
825 def p4BranchesInGit(branchesAreInRemotes=True):
826     """Find all the branches whose names start with "p4/", looking
827        in remotes or heads as specified by the argument.  Return
828        a dictionary of { branch: revision } for each one found.
829        The branch names are the short names, without any
830        "p4/" prefix."""
831
832     branches = {}
833
834     cmdline = "git rev-parse --symbolic "
835     if branchesAreInRemotes:
836         cmdline += "--remotes"
837     else:
838         cmdline += "--branches"
839
840     for line in read_pipe_lines(cmdline):
841         line = line.strip()
842
843         # only import to p4/
844         if not line.startswith('p4/'):
845             continue
846         # special symbolic ref to p4/master
847         if line == "p4/HEAD":
848             continue
849
850         # strip off p4/ prefix
851         branch = line[len("p4/"):]
852
853         branches[branch] = parseRevision(line)
854
855     return branches
856
857 def branch_exists(branch):
858     """Make sure that the given ref name really exists."""
859
860     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
861     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
862     out, _ = p.communicate()
863     if p.returncode:
864         return False
865     # expect exactly one line of output: the branch name
866     return out.rstrip() == branch
867
868 def findUpstreamBranchPoint(head = "HEAD"):
869     branches = p4BranchesInGit()
870     # map from depot-path to branch name
871     branchByDepotPath = {}
872     for branch in branches.keys():
873         tip = branches[branch]
874         log = extractLogMessageFromGitCommit(tip)
875         settings = extractSettingsGitLog(log)
876         if "depot-paths" in settings:
877             paths = ",".join(settings["depot-paths"])
878             branchByDepotPath[paths] = "remotes/p4/" + branch
879
880     settings = None
881     parent = 0
882     while parent < 65535:
883         commit = head + "~%s" % parent
884         log = extractLogMessageFromGitCommit(commit)
885         settings = extractSettingsGitLog(log)
886         if "depot-paths" in settings:
887             paths = ",".join(settings["depot-paths"])
888             if paths in branchByDepotPath:
889                 return [branchByDepotPath[paths], settings]
890
891         parent = parent + 1
892
893     return ["", settings]
894
895 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
896     if not silent:
897         print("Creating/updating branch(es) in %s based on origin branch(es)"
898                % localRefPrefix)
899
900     originPrefix = "origin/p4/"
901
902     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
903         line = line.strip()
904         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
905             continue
906
907         headName = line[len(originPrefix):]
908         remoteHead = localRefPrefix + headName
909         originHead = line
910
911         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
912         if ('depot-paths' not in original
913             or 'change' not in original):
914             continue
915
916         update = False
917         if not gitBranchExists(remoteHead):
918             if verbose:
919                 print("creating %s" % remoteHead)
920             update = True
921         else:
922             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
923             if 'change' in settings:
924                 if settings['depot-paths'] == original['depot-paths']:
925                     originP4Change = int(original['change'])
926                     p4Change = int(settings['change'])
927                     if originP4Change > p4Change:
928                         print("%s (%s) is newer than %s (%s). "
929                                "Updating p4 branch from origin."
930                                % (originHead, originP4Change,
931                                   remoteHead, p4Change))
932                         update = True
933                 else:
934                     print("Ignoring: %s was imported from %s while "
935                            "%s was imported from %s"
936                            % (originHead, ','.join(original['depot-paths']),
937                               remoteHead, ','.join(settings['depot-paths'])))
938
939         if update:
940             system("git update-ref %s %s" % (remoteHead, originHead))
941
942 def originP4BranchesExist():
943         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
944
945
946 def p4ParseNumericChangeRange(parts):
947     changeStart = int(parts[0][1:])
948     if parts[1] == '#head':
949         changeEnd = p4_last_change()
950     else:
951         changeEnd = int(parts[1])
952
953     return (changeStart, changeEnd)
954
955 def chooseBlockSize(blockSize):
956     if blockSize:
957         return blockSize
958     else:
959         return defaultBlockSize
960
961 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
962     assert depotPaths
963
964     # Parse the change range into start and end. Try to find integer
965     # revision ranges as these can be broken up into blocks to avoid
966     # hitting server-side limits (maxrows, maxscanresults). But if
967     # that doesn't work, fall back to using the raw revision specifier
968     # strings, without using block mode.
969
970     if changeRange is None or changeRange == '':
971         changeStart = 1
972         changeEnd = p4_last_change()
973         block_size = chooseBlockSize(requestedBlockSize)
974     else:
975         parts = changeRange.split(',')
976         assert len(parts) == 2
977         try:
978             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
979             block_size = chooseBlockSize(requestedBlockSize)
980         except ValueError:
981             changeStart = parts[0][1:]
982             changeEnd = parts[1]
983             if requestedBlockSize:
984                 die("cannot use --changes-block-size with non-numeric revisions")
985             block_size = None
986
987     changes = set()
988
989     # Retrieve changes a block at a time, to prevent running
990     # into a MaxResults/MaxScanRows error from the server. If
991     # we _do_ hit one of those errors, turn down the block size
992
993     while True:
994         cmd = ['changes']
995
996         if block_size:
997             end = min(changeEnd, changeStart + block_size)
998             revisionRange = "%d,%d" % (changeStart, end)
999         else:
1000             revisionRange = "%s,%s" % (changeStart, changeEnd)
1001
1002         for p in depotPaths:
1003             cmd += ["%s...@%s" % (p, revisionRange)]
1004
1005         # fetch the changes
1006         try:
1007             result = p4CmdList(cmd, errors_as_exceptions=True)
1008         except P4RequestSizeException as e:
1009             if not block_size:
1010                 block_size = e.limit
1011             elif block_size > e.limit:
1012                 block_size = e.limit
1013             else:
1014                 block_size = max(2, block_size // 2)
1015
1016             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1017             continue
1018         except P4Exception as e:
1019             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1020
1021         # Insert changes in chronological order
1022         for entry in reversed(result):
1023             if 'change' not in entry:
1024                 continue
1025             changes.add(int(entry['change']))
1026
1027         if not block_size:
1028             break
1029
1030         if end >= changeEnd:
1031             break
1032
1033         changeStart = end + 1
1034
1035     changes = sorted(changes)
1036     return changes
1037
1038 def p4PathStartsWith(path, prefix):
1039     # This method tries to remedy a potential mixed-case issue:
1040     #
1041     # If UserA adds  //depot/DirA/file1
1042     # and UserB adds //depot/dira/file2
1043     #
1044     # we may or may not have a problem. If you have core.ignorecase=true,
1045     # we treat DirA and dira as the same directory
1046     if gitConfigBool("core.ignorecase"):
1047         return path.lower().startswith(prefix.lower())
1048     return path.startswith(prefix)
1049
1050 def getClientSpec():
1051     """Look at the p4 client spec, create a View() object that contains
1052        all the mappings, and return it."""
1053
1054     specList = p4CmdList("client -o")
1055     if len(specList) != 1:
1056         die('Output from "client -o" is %d lines, expecting 1' %
1057             len(specList))
1058
1059     # dictionary of all client parameters
1060     entry = specList[0]
1061
1062     # the //client/ name
1063     client_name = entry["Client"]
1064
1065     # just the keys that start with "View"
1066     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1067
1068     # hold this new View
1069     view = View(client_name)
1070
1071     # append the lines, in order, to the view
1072     for view_num in range(len(view_keys)):
1073         k = "View%d" % view_num
1074         if k not in view_keys:
1075             die("Expected view key %s missing" % k)
1076         view.append(entry[k])
1077
1078     return view
1079
1080 def getClientRoot():
1081     """Grab the client directory."""
1082
1083     output = p4CmdList("client -o")
1084     if len(output) != 1:
1085         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1086
1087     entry = output[0]
1088     if "Root" not in entry:
1089         die('Client has no "Root"')
1090
1091     return entry["Root"]
1092
1093 #
1094 # P4 wildcards are not allowed in filenames.  P4 complains
1095 # if you simply add them, but you can force it with "-f", in
1096 # which case it translates them into %xx encoding internally.
1097 #
1098 def wildcard_decode(path):
1099     # Search for and fix just these four characters.  Do % last so
1100     # that fixing it does not inadvertently create new %-escapes.
1101     # Cannot have * in a filename in windows; untested as to
1102     # what p4 would do in such a case.
1103     if not platform.system() == "Windows":
1104         path = path.replace("%2A", "*")
1105     path = path.replace("%23", "#") \
1106                .replace("%40", "@") \
1107                .replace("%25", "%")
1108     return path
1109
1110 def wildcard_encode(path):
1111     # do % first to avoid double-encoding the %s introduced here
1112     path = path.replace("%", "%25") \
1113                .replace("*", "%2A") \
1114                .replace("#", "%23") \
1115                .replace("@", "%40")
1116     return path
1117
1118 def wildcard_present(path):
1119     m = re.search("[*#@%]", path)
1120     return m is not None
1121
1122 class LargeFileSystem(object):
1123     """Base class for large file system support."""
1124
1125     def __init__(self, writeToGitStream):
1126         self.largeFiles = set()
1127         self.writeToGitStream = writeToGitStream
1128
1129     def generatePointer(self, cloneDestination, contentFile):
1130         """Return the content of a pointer file that is stored in Git instead of
1131            the actual content."""
1132         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1133
1134     def pushFile(self, localLargeFile):
1135         """Push the actual content which is not stored in the Git repository to
1136            a server."""
1137         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1138
1139     def hasLargeFileExtension(self, relPath):
1140         return reduce(
1141             lambda a, b: a or b,
1142             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1143             False
1144         )
1145
1146     def generateTempFile(self, contents):
1147         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1148         for d in contents:
1149             contentFile.write(d)
1150         contentFile.close()
1151         return contentFile.name
1152
1153     def exceedsLargeFileThreshold(self, relPath, contents):
1154         if gitConfigInt('git-p4.largeFileThreshold'):
1155             contentsSize = sum(len(d) for d in contents)
1156             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1157                 return True
1158         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159             contentsSize = sum(len(d) for d in contents)
1160             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1161                 return False
1162             contentTempFile = self.generateTempFile(contents)
1163             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1164             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1165             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1166             zf.close()
1167             compressedContentsSize = zf.infolist()[0].compress_size
1168             os.remove(contentTempFile)
1169             os.remove(compressedContentFile.name)
1170             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1171                 return True
1172         return False
1173
1174     def addLargeFile(self, relPath):
1175         self.largeFiles.add(relPath)
1176
1177     def removeLargeFile(self, relPath):
1178         self.largeFiles.remove(relPath)
1179
1180     def isLargeFile(self, relPath):
1181         return relPath in self.largeFiles
1182
1183     def processContent(self, git_mode, relPath, contents):
1184         """Processes the content of git fast import. This method decides if a
1185            file is stored in the large file system and handles all necessary
1186            steps."""
1187         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1188             contentTempFile = self.generateTempFile(contents)
1189             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1190             if pointer_git_mode:
1191                 git_mode = pointer_git_mode
1192             if localLargeFile:
1193                 # Move temp file to final location in large file system
1194                 largeFileDir = os.path.dirname(localLargeFile)
1195                 if not os.path.isdir(largeFileDir):
1196                     os.makedirs(largeFileDir)
1197                 shutil.move(contentTempFile, localLargeFile)
1198                 self.addLargeFile(relPath)
1199                 if gitConfigBool('git-p4.largeFilePush'):
1200                     self.pushFile(localLargeFile)
1201                 if verbose:
1202                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1203         return (git_mode, contents)
1204
1205 class MockLFS(LargeFileSystem):
1206     """Mock large file system for testing."""
1207
1208     def generatePointer(self, contentFile):
1209         """The pointer content is the original content prefixed with "pointer-".
1210            The local filename of the large file storage is derived from the file content.
1211            """
1212         with open(contentFile, 'r') as f:
1213             content = next(f)
1214             gitMode = '100644'
1215             pointerContents = 'pointer-' + content
1216             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1217             return (gitMode, pointerContents, localLargeFile)
1218
1219     def pushFile(self, localLargeFile):
1220         """The remote filename of the large file storage is the same as the local
1221            one but in a different directory.
1222            """
1223         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1224         if not os.path.exists(remotePath):
1225             os.makedirs(remotePath)
1226         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1227
1228 class GitLFS(LargeFileSystem):
1229     """Git LFS as backend for the git-p4 large file system.
1230        See https://git-lfs.github.com/ for details."""
1231
1232     def __init__(self, *args):
1233         LargeFileSystem.__init__(self, *args)
1234         self.baseGitAttributes = []
1235
1236     def generatePointer(self, contentFile):
1237         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1238            mode and content which is stored in the Git repository instead of
1239            the actual content. Return also the new location of the actual
1240            content.
1241            """
1242         if os.path.getsize(contentFile) == 0:
1243             return (None, '', None)
1244
1245         pointerProcess = subprocess.Popen(
1246             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1247             stdout=subprocess.PIPE
1248         )
1249         pointerFile = pointerProcess.stdout.read()
1250         if pointerProcess.wait():
1251             os.remove(contentFile)
1252             die('git-lfs pointer command failed. Did you install the extension?')
1253
1254         # Git LFS removed the preamble in the output of the 'pointer' command
1255         # starting from version 1.2.0. Check for the preamble here to support
1256         # earlier versions.
1257         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1258         if pointerFile.startswith('Git LFS pointer for'):
1259             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1260
1261         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1262         localLargeFile = os.path.join(
1263             os.getcwd(),
1264             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1265             oid,
1266         )
1267         # LFS Spec states that pointer files should not have the executable bit set.
1268         gitMode = '100644'
1269         return (gitMode, pointerFile, localLargeFile)
1270
1271     def pushFile(self, localLargeFile):
1272         uploadProcess = subprocess.Popen(
1273             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1274         )
1275         if uploadProcess.wait():
1276             die('git-lfs push command failed. Did you define a remote?')
1277
1278     def generateGitAttributes(self):
1279         return (
1280             self.baseGitAttributes +
1281             [
1282                 '\n',
1283                 '#\n',
1284                 '# Git LFS (see https://git-lfs.github.com/)\n',
1285                 '#\n',
1286             ] +
1287             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1288                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1289             ] +
1290             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1291                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1292             ]
1293         )
1294
1295     def addLargeFile(self, relPath):
1296         LargeFileSystem.addLargeFile(self, relPath)
1297         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1298
1299     def removeLargeFile(self, relPath):
1300         LargeFileSystem.removeLargeFile(self, relPath)
1301         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1302
1303     def processContent(self, git_mode, relPath, contents):
1304         if relPath == '.gitattributes':
1305             self.baseGitAttributes = contents
1306             return (git_mode, self.generateGitAttributes())
1307         else:
1308             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1309
1310 class Command:
1311     delete_actions = ( "delete", "move/delete", "purge" )
1312     add_actions = ( "add", "branch", "move/add" )
1313
1314     def __init__(self):
1315         self.usage = "usage: %prog [options]"
1316         self.needsGit = True
1317         self.verbose = False
1318
1319     # This is required for the "append" update_shelve action
1320     def ensure_value(self, attr, value):
1321         if not hasattr(self, attr) or getattr(self, attr) is None:
1322             setattr(self, attr, value)
1323         return getattr(self, attr)
1324
1325 class P4UserMap:
1326     def __init__(self):
1327         self.userMapFromPerforceServer = False
1328         self.myP4UserId = None
1329
1330     def p4UserId(self):
1331         if self.myP4UserId:
1332             return self.myP4UserId
1333
1334         results = p4CmdList("user -o")
1335         for r in results:
1336             if 'User' in r:
1337                 self.myP4UserId = r['User']
1338                 return r['User']
1339         die("Could not find your p4 user id")
1340
1341     def p4UserIsMe(self, p4User):
1342         # return True if the given p4 user is actually me
1343         me = self.p4UserId()
1344         if not p4User or p4User != me:
1345             return False
1346         else:
1347             return True
1348
1349     def getUserCacheFilename(self):
1350         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1351         return home + "/.gitp4-usercache.txt"
1352
1353     def getUserMapFromPerforceServer(self):
1354         if self.userMapFromPerforceServer:
1355             return
1356         self.users = {}
1357         self.emails = {}
1358
1359         for output in p4CmdList("users"):
1360             if "User" not in output:
1361                 continue
1362             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1363             self.emails[output["Email"]] = output["User"]
1364
1365         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1366         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1367             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1368             if mapUser and len(mapUser[0]) == 3:
1369                 user = mapUser[0][0]
1370                 fullname = mapUser[0][1]
1371                 email = mapUser[0][2]
1372                 self.users[user] = fullname + " <" + email + ">"
1373                 self.emails[email] = user
1374
1375         s = ''
1376         for (key, val) in self.users.items():
1377             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1378
1379         open(self.getUserCacheFilename(), "wb").write(s)
1380         self.userMapFromPerforceServer = True
1381
1382     def loadUserMapFromCache(self):
1383         self.users = {}
1384         self.userMapFromPerforceServer = False
1385         try:
1386             cache = open(self.getUserCacheFilename(), "rb")
1387             lines = cache.readlines()
1388             cache.close()
1389             for line in lines:
1390                 entry = line.strip().split("\t")
1391                 self.users[entry[0]] = entry[1]
1392         except IOError:
1393             self.getUserMapFromPerforceServer()
1394
1395 class P4Debug(Command):
1396     def __init__(self):
1397         Command.__init__(self)
1398         self.options = []
1399         self.description = "A tool to debug the output of p4 -G."
1400         self.needsGit = False
1401
1402     def run(self, args):
1403         j = 0
1404         for output in p4CmdList(args):
1405             print('Element: %d' % j)
1406             j += 1
1407             print(output)
1408         return True
1409
1410 class P4RollBack(Command):
1411     def __init__(self):
1412         Command.__init__(self)
1413         self.options = [
1414             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1415         ]
1416         self.description = "A tool to debug the multi-branch import. Don't use :)"
1417         self.rollbackLocalBranches = False
1418
1419     def run(self, args):
1420         if len(args) != 1:
1421             return False
1422         maxChange = int(args[0])
1423
1424         if "p4ExitCode" in p4Cmd("changes -m 1"):
1425             die("Problems executing p4");
1426
1427         if self.rollbackLocalBranches:
1428             refPrefix = "refs/heads/"
1429             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1430         else:
1431             refPrefix = "refs/remotes/"
1432             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1433
1434         for line in lines:
1435             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1436                 line = line.strip()
1437                 ref = refPrefix + line
1438                 log = extractLogMessageFromGitCommit(ref)
1439                 settings = extractSettingsGitLog(log)
1440
1441                 depotPaths = settings['depot-paths']
1442                 change = settings['change']
1443
1444                 changed = False
1445
1446                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1447                                                            for p in depotPaths]))) == 0:
1448                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1449                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1450                     continue
1451
1452                 while change and int(change) > maxChange:
1453                     changed = True
1454                     if self.verbose:
1455                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1456                     system("git update-ref %s \"%s^\"" % (ref, ref))
1457                     log = extractLogMessageFromGitCommit(ref)
1458                     settings =  extractSettingsGitLog(log)
1459
1460
1461                     depotPaths = settings['depot-paths']
1462                     change = settings['change']
1463
1464                 if changed:
1465                     print("%s rewound to %s" % (ref, change))
1466
1467         return True
1468
1469 class P4Submit(Command, P4UserMap):
1470
1471     conflict_behavior_choices = ("ask", "skip", "quit")
1472
1473     def __init__(self):
1474         Command.__init__(self)
1475         P4UserMap.__init__(self)
1476         self.options = [
1477                 optparse.make_option("--origin", dest="origin"),
1478                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1479                 # preserve the user, requires relevant p4 permissions
1480                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1481                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1482                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1483                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1484                 optparse.make_option("--conflict", dest="conflict_behavior",
1485                                      choices=self.conflict_behavior_choices),
1486                 optparse.make_option("--branch", dest="branch"),
1487                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1488                                      help="Shelve instead of submit. Shelved files are reverted, "
1489                                      "restoring the workspace to the state before the shelve"),
1490                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1491                                      metavar="CHANGELIST",
1492                                      help="update an existing shelved changelist, implies --shelve, "
1493                                            "repeat in-order for multiple shelved changelists"),
1494                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1495                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1496                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1497                                      help="Disable rebase after submit is completed. Can be useful if you "
1498                                      "work from a local git branch that is not master"),
1499                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1500                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1501         ]
1502         self.description = """Submit changes from git to the perforce depot.\n
1503     The `p4-pre-submit` hook is executed if it exists and is executable.
1504     The hook takes no parameters and nothing from standard input. Exiting with
1505     non-zero status from this script prevents `git-p4 submit` from launching.
1506
1507     One usage scenario is to run unit tests in the hook."""
1508
1509         self.usage += " [name of git branch to submit into perforce depot]"
1510         self.origin = ""
1511         self.detectRenames = False
1512         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1513         self.dry_run = False
1514         self.shelve = False
1515         self.update_shelve = list()
1516         self.commit = ""
1517         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1518         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1519         self.prepare_p4_only = False
1520         self.conflict_behavior = None
1521         self.isWindows = (platform.system() == "Windows")
1522         self.exportLabels = False
1523         self.p4HasMoveCommand = p4_has_move_command()
1524         self.branch = None
1525
1526         if gitConfig('git-p4.largeFileSystem'):
1527             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1528
1529     def check(self):
1530         if len(p4CmdList("opened ...")) > 0:
1531             die("You have files opened with perforce! Close them before starting the sync.")
1532
1533     def separate_jobs_from_description(self, message):
1534         """Extract and return a possible Jobs field in the commit
1535            message.  It goes into a separate section in the p4 change
1536            specification.
1537
1538            A jobs line starts with "Jobs:" and looks like a new field
1539            in a form.  Values are white-space separated on the same
1540            line or on following lines that start with a tab.
1541
1542            This does not parse and extract the full git commit message
1543            like a p4 form.  It just sees the Jobs: line as a marker
1544            to pass everything from then on directly into the p4 form,
1545            but outside the description section.
1546
1547            Return a tuple (stripped log message, jobs string)."""
1548
1549         m = re.search(r'^Jobs:', message, re.MULTILINE)
1550         if m is None:
1551             return (message, None)
1552
1553         jobtext = message[m.start():]
1554         stripped_message = message[:m.start()].rstrip()
1555         return (stripped_message, jobtext)
1556
1557     def prepareLogMessage(self, template, message, jobs):
1558         """Edits the template returned from "p4 change -o" to insert
1559            the message in the Description field, and the jobs text in
1560            the Jobs field."""
1561         result = ""
1562
1563         inDescriptionSection = False
1564
1565         for line in template.split("\n"):
1566             if line.startswith("#"):
1567                 result += line + "\n"
1568                 continue
1569
1570             if inDescriptionSection:
1571                 if line.startswith("Files:") or line.startswith("Jobs:"):
1572                     inDescriptionSection = False
1573                     # insert Jobs section
1574                     if jobs:
1575                         result += jobs + "\n"
1576                 else:
1577                     continue
1578             else:
1579                 if line.startswith("Description:"):
1580                     inDescriptionSection = True
1581                     line += "\n"
1582                     for messageLine in message.split("\n"):
1583                         line += "\t" + messageLine + "\n"
1584
1585             result += line + "\n"
1586
1587         return result
1588
1589     def patchRCSKeywords(self, file, pattern):
1590         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1591         (handle, outFileName) = tempfile.mkstemp(dir='.')
1592         try:
1593             outFile = os.fdopen(handle, "w+")
1594             inFile = open(file, "r")
1595             regexp = re.compile(pattern, re.VERBOSE)
1596             for line in inFile.readlines():
1597                 line = regexp.sub(r'$\1$', line)
1598                 outFile.write(line)
1599             inFile.close()
1600             outFile.close()
1601             # Forcibly overwrite the original file
1602             os.unlink(file)
1603             shutil.move(outFileName, file)
1604         except:
1605             # cleanup our temporary file
1606             os.unlink(outFileName)
1607             print("Failed to strip RCS keywords in %s" % file)
1608             raise
1609
1610         print("Patched up RCS keywords in %s" % file)
1611
1612     def p4UserForCommit(self,id):
1613         # Return the tuple (perforce user,git email) for a given git commit id
1614         self.getUserMapFromPerforceServer()
1615         gitEmail = read_pipe(["git", "log", "--max-count=1",
1616                               "--format=%ae", id])
1617         gitEmail = gitEmail.strip()
1618         if gitEmail not in self.emails:
1619             return (None,gitEmail)
1620         else:
1621             return (self.emails[gitEmail],gitEmail)
1622
1623     def checkValidP4Users(self,commits):
1624         # check if any git authors cannot be mapped to p4 users
1625         for id in commits:
1626             (user,email) = self.p4UserForCommit(id)
1627             if not user:
1628                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1629                 if gitConfigBool("git-p4.allowMissingP4Users"):
1630                     print("%s" % msg)
1631                 else:
1632                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1633
1634     def lastP4Changelist(self):
1635         # Get back the last changelist number submitted in this client spec. This
1636         # then gets used to patch up the username in the change. If the same
1637         # client spec is being used by multiple processes then this might go
1638         # wrong.
1639         results = p4CmdList("client -o")        # find the current client
1640         client = None
1641         for r in results:
1642             if 'Client' in r:
1643                 client = r['Client']
1644                 break
1645         if not client:
1646             die("could not get client spec")
1647         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1648         for r in results:
1649             if 'change' in r:
1650                 return r['change']
1651         die("Could not get changelist number for last submit - cannot patch up user details")
1652
1653     def modifyChangelistUser(self, changelist, newUser):
1654         # fixup the user field of a changelist after it has been submitted.
1655         changes = p4CmdList("change -o %s" % changelist)
1656         if len(changes) != 1:
1657             die("Bad output from p4 change modifying %s to user %s" %
1658                 (changelist, newUser))
1659
1660         c = changes[0]
1661         if c['User'] == newUser: return   # nothing to do
1662         c['User'] = newUser
1663         input = marshal.dumps(c)
1664
1665         result = p4CmdList("change -f -i", stdin=input)
1666         for r in result:
1667             if 'code' in r:
1668                 if r['code'] == 'error':
1669                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1670             if 'data' in r:
1671                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1672                 return
1673         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1674
1675     def canChangeChangelists(self):
1676         # check to see if we have p4 admin or super-user permissions, either of
1677         # which are required to modify changelists.
1678         results = p4CmdList(["protects", self.depotPath])
1679         for r in results:
1680             if 'perm' in r:
1681                 if r['perm'] == 'admin':
1682                     return 1
1683                 if r['perm'] == 'super':
1684                     return 1
1685         return 0
1686
1687     def prepareSubmitTemplate(self, changelist=None):
1688         """Run "p4 change -o" to grab a change specification template.
1689            This does not use "p4 -G", as it is nice to keep the submission
1690            template in original order, since a human might edit it.
1691
1692            Remove lines in the Files section that show changes to files
1693            outside the depot path we're committing into."""
1694
1695         [upstream, settings] = findUpstreamBranchPoint()
1696
1697         template = """\
1698 # A Perforce Change Specification.
1699 #
1700 #  Change:      The change number. 'new' on a new changelist.
1701 #  Date:        The date this specification was last modified.
1702 #  Client:      The client on which the changelist was created.  Read-only.
1703 #  User:        The user who created the changelist.
1704 #  Status:      Either 'pending' or 'submitted'. Read-only.
1705 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1706 #  Description: Comments about the changelist.  Required.
1707 #  Jobs:        What opened jobs are to be closed by this changelist.
1708 #               You may delete jobs from this list.  (New changelists only.)
1709 #  Files:       What opened files from the default changelist are to be added
1710 #               to this changelist.  You may delete files from this list.
1711 #               (New changelists only.)
1712 """
1713         files_list = []
1714         inFilesSection = False
1715         change_entry = None
1716         args = ['change', '-o']
1717         if changelist:
1718             args.append(str(changelist))
1719         for entry in p4CmdList(args):
1720             if 'code' not in entry:
1721                 continue
1722             if entry['code'] == 'stat':
1723                 change_entry = entry
1724                 break
1725         if not change_entry:
1726             die('Failed to decode output of p4 change -o')
1727         for key, value in change_entry.iteritems():
1728             if key.startswith('File'):
1729                 if 'depot-paths' in settings:
1730                     if not [p for p in settings['depot-paths']
1731                             if p4PathStartsWith(value, p)]:
1732                         continue
1733                 else:
1734                     if not p4PathStartsWith(value, self.depotPath):
1735                         continue
1736                 files_list.append(value)
1737                 continue
1738         # Output in the order expected by prepareLogMessage
1739         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1740             if key not in change_entry:
1741                 continue
1742             template += '\n'
1743             template += key + ':'
1744             if key == 'Description':
1745                 template += '\n'
1746             for field_line in change_entry[key].splitlines():
1747                 template += '\t'+field_line+'\n'
1748         if len(files_list) > 0:
1749             template += '\n'
1750             template += 'Files:\n'
1751         for path in files_list:
1752             template += '\t'+path+'\n'
1753         return template
1754
1755     def edit_template(self, template_file):
1756         """Invoke the editor to let the user change the submission
1757            message.  Return true if okay to continue with the submit."""
1758
1759         # if configured to skip the editing part, just submit
1760         if gitConfigBool("git-p4.skipSubmitEdit"):
1761             return True
1762
1763         # look at the modification time, to check later if the user saved
1764         # the file
1765         mtime = os.stat(template_file).st_mtime
1766
1767         # invoke the editor
1768         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1769             editor = os.environ.get("P4EDITOR")
1770         else:
1771             editor = read_pipe("git var GIT_EDITOR").strip()
1772         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1773
1774         # If the file was not saved, prompt to see if this patch should
1775         # be skipped.  But skip this verification step if configured so.
1776         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1777             return True
1778
1779         # modification time updated means user saved the file
1780         if os.stat(template_file).st_mtime > mtime:
1781             return True
1782
1783         while True:
1784             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1785             if response == 'y':
1786                 return True
1787             if response == 'n':
1788                 return False
1789
1790     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1791         # diff
1792         if "P4DIFF" in os.environ:
1793             del(os.environ["P4DIFF"])
1794         diff = ""
1795         for editedFile in editedFiles:
1796             diff += p4_read_pipe(['diff', '-du',
1797                                   wildcard_encode(editedFile)])
1798
1799         # new file diff
1800         newdiff = ""
1801         for newFile in filesToAdd:
1802             newdiff += "==== new file ====\n"
1803             newdiff += "--- /dev/null\n"
1804             newdiff += "+++ %s\n" % newFile
1805
1806             is_link = os.path.islink(newFile)
1807             expect_link = newFile in symlinks
1808
1809             if is_link and expect_link:
1810                 newdiff += "+%s\n" % os.readlink(newFile)
1811             else:
1812                 f = open(newFile, "r")
1813                 for line in f.readlines():
1814                     newdiff += "+" + line
1815                 f.close()
1816
1817         return (diff + newdiff).replace('\r\n', '\n')
1818
1819     def applyCommit(self, id):
1820         """Apply one commit, return True if it succeeded."""
1821
1822         print("Applying", read_pipe(["git", "show", "-s",
1823                                      "--format=format:%h %s", id]))
1824
1825         (p4User, gitEmail) = self.p4UserForCommit(id)
1826
1827         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1828         filesToAdd = set()
1829         filesToChangeType = set()
1830         filesToDelete = set()
1831         editedFiles = set()
1832         pureRenameCopy = set()
1833         symlinks = set()
1834         filesToChangeExecBit = {}
1835         all_files = list()
1836
1837         for line in diff:
1838             diff = parseDiffTreeEntry(line)
1839             modifier = diff['status']
1840             path = diff['src']
1841             all_files.append(path)
1842
1843             if modifier == "M":
1844                 p4_edit(path)
1845                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1846                     filesToChangeExecBit[path] = diff['dst_mode']
1847                 editedFiles.add(path)
1848             elif modifier == "A":
1849                 filesToAdd.add(path)
1850                 filesToChangeExecBit[path] = diff['dst_mode']
1851                 if path in filesToDelete:
1852                     filesToDelete.remove(path)
1853
1854                 dst_mode = int(diff['dst_mode'], 8)
1855                 if dst_mode == 0o120000:
1856                     symlinks.add(path)
1857
1858             elif modifier == "D":
1859                 filesToDelete.add(path)
1860                 if path in filesToAdd:
1861                     filesToAdd.remove(path)
1862             elif modifier == "C":
1863                 src, dest = diff['src'], diff['dst']
1864                 all_files.append(dest)
1865                 p4_integrate(src, dest)
1866                 pureRenameCopy.add(dest)
1867                 if diff['src_sha1'] != diff['dst_sha1']:
1868                     p4_edit(dest)
1869                     pureRenameCopy.discard(dest)
1870                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1871                     p4_edit(dest)
1872                     pureRenameCopy.discard(dest)
1873                     filesToChangeExecBit[dest] = diff['dst_mode']
1874                 if self.isWindows:
1875                     # turn off read-only attribute
1876                     os.chmod(dest, stat.S_IWRITE)
1877                 os.unlink(dest)
1878                 editedFiles.add(dest)
1879             elif modifier == "R":
1880                 src, dest = diff['src'], diff['dst']
1881                 all_files.append(dest)
1882                 if self.p4HasMoveCommand:
1883                     p4_edit(src)        # src must be open before move
1884                     p4_move(src, dest)  # opens for (move/delete, move/add)
1885                 else:
1886                     p4_integrate(src, dest)
1887                     if diff['src_sha1'] != diff['dst_sha1']:
1888                         p4_edit(dest)
1889                     else:
1890                         pureRenameCopy.add(dest)
1891                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1892                     if not self.p4HasMoveCommand:
1893                         p4_edit(dest)   # with move: already open, writable
1894                     filesToChangeExecBit[dest] = diff['dst_mode']
1895                 if not self.p4HasMoveCommand:
1896                     if self.isWindows:
1897                         os.chmod(dest, stat.S_IWRITE)
1898                     os.unlink(dest)
1899                     filesToDelete.add(src)
1900                 editedFiles.add(dest)
1901             elif modifier == "T":
1902                 filesToChangeType.add(path)
1903             else:
1904                 die("unknown modifier %s for %s" % (modifier, path))
1905
1906         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1907         patchcmd = diffcmd + " | git apply "
1908         tryPatchCmd = patchcmd + "--check -"
1909         applyPatchCmd = patchcmd + "--check --apply -"
1910         patch_succeeded = True
1911
1912         if os.system(tryPatchCmd) != 0:
1913             fixed_rcs_keywords = False
1914             patch_succeeded = False
1915             print("Unfortunately applying the change failed!")
1916
1917             # Patch failed, maybe it's just RCS keyword woes. Look through
1918             # the patch to see if that's possible.
1919             if gitConfigBool("git-p4.attemptRCSCleanup"):
1920                 file = None
1921                 pattern = None
1922                 kwfiles = {}
1923                 for file in editedFiles | filesToDelete:
1924                     # did this file's delta contain RCS keywords?
1925                     pattern = p4_keywords_regexp_for_file(file)
1926
1927                     if pattern:
1928                         # this file is a possibility...look for RCS keywords.
1929                         regexp = re.compile(pattern, re.VERBOSE)
1930                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1931                             if regexp.search(line):
1932                                 if verbose:
1933                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1934                                 kwfiles[file] = pattern
1935                                 break
1936
1937                 for file in kwfiles:
1938                     if verbose:
1939                         print("zapping %s with %s" % (line,pattern))
1940                     # File is being deleted, so not open in p4.  Must
1941                     # disable the read-only bit on windows.
1942                     if self.isWindows and file not in editedFiles:
1943                         os.chmod(file, stat.S_IWRITE)
1944                     self.patchRCSKeywords(file, kwfiles[file])
1945                     fixed_rcs_keywords = True
1946
1947             if fixed_rcs_keywords:
1948                 print("Retrying the patch with RCS keywords cleaned up")
1949                 if os.system(tryPatchCmd) == 0:
1950                     patch_succeeded = True
1951
1952         if not patch_succeeded:
1953             for f in editedFiles:
1954                 p4_revert(f)
1955             return False
1956
1957         #
1958         # Apply the patch for real, and do add/delete/+x handling.
1959         #
1960         system(applyPatchCmd)
1961
1962         for f in filesToChangeType:
1963             p4_edit(f, "-t", "auto")
1964         for f in filesToAdd:
1965             p4_add(f)
1966         for f in filesToDelete:
1967             p4_revert(f)
1968             p4_delete(f)
1969
1970         # Set/clear executable bits
1971         for f in filesToChangeExecBit.keys():
1972             mode = filesToChangeExecBit[f]
1973             setP4ExecBit(f, mode)
1974
1975         update_shelve = 0
1976         if len(self.update_shelve) > 0:
1977             update_shelve = self.update_shelve.pop(0)
1978             p4_reopen_in_change(update_shelve, all_files)
1979
1980         #
1981         # Build p4 change description, starting with the contents
1982         # of the git commit message.
1983         #
1984         logMessage = extractLogMessageFromGitCommit(id)
1985         logMessage = logMessage.strip()
1986         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1987
1988         template = self.prepareSubmitTemplate(update_shelve)
1989         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1990
1991         if self.preserveUser:
1992            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1993
1994         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1995             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1996             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1997             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1998
1999         separatorLine = "######## everything below this line is just the diff #######\n"
2000         if not self.prepare_p4_only:
2001             submitTemplate += separatorLine
2002             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2003
2004         (handle, fileName) = tempfile.mkstemp()
2005         tmpFile = os.fdopen(handle, "w+b")
2006         if self.isWindows:
2007             submitTemplate = submitTemplate.replace("\n", "\r\n")
2008         tmpFile.write(submitTemplate)
2009         tmpFile.close()
2010
2011         if self.prepare_p4_only:
2012             #
2013             # Leave the p4 tree prepared, and the submit template around
2014             # and let the user decide what to do next
2015             #
2016             print()
2017             print("P4 workspace prepared for submission.")
2018             print("To submit or revert, go to client workspace")
2019             print("  " + self.clientPath)
2020             print()
2021             print("To submit, use \"p4 submit\" to write a new description,")
2022             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2023                   " \"git p4\"." % fileName)
2024             print("You can delete the file \"%s\" when finished." % fileName)
2025
2026             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2027                 print("To preserve change ownership by user %s, you must\n" \
2028                       "do \"p4 change -f <change>\" after submitting and\n" \
2029                       "edit the User field.")
2030             if pureRenameCopy:
2031                 print("After submitting, renamed files must be re-synced.")
2032                 print("Invoke \"p4 sync -f\" on each of these files:")
2033                 for f in pureRenameCopy:
2034                     print("  " + f)
2035
2036             print()
2037             print("To revert the changes, use \"p4 revert ...\", and delete")
2038             print("the submit template file \"%s\"" % fileName)
2039             if filesToAdd:
2040                 print("Since the commit adds new files, they must be deleted:")
2041                 for f in filesToAdd:
2042                     print("  " + f)
2043             print()
2044             return True
2045
2046         #
2047         # Let the user edit the change description, then submit it.
2048         #
2049         submitted = False
2050
2051         try:
2052             if self.edit_template(fileName):
2053                 # read the edited message and submit
2054                 tmpFile = open(fileName, "rb")
2055                 message = tmpFile.read()
2056                 tmpFile.close()
2057                 if self.isWindows:
2058                     message = message.replace("\r\n", "\n")
2059                 submitTemplate = message[:message.index(separatorLine)]
2060
2061                 if update_shelve:
2062                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2063                 elif self.shelve:
2064                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2065                 else:
2066                     p4_write_pipe(['submit', '-i'], submitTemplate)
2067                     # The rename/copy happened by applying a patch that created a
2068                     # new file.  This leaves it writable, which confuses p4.
2069                     for f in pureRenameCopy:
2070                         p4_sync(f, "-f")
2071
2072                 if self.preserveUser:
2073                     if p4User:
2074                         # Get last changelist number. Cannot easily get it from
2075                         # the submit command output as the output is
2076                         # unmarshalled.
2077                         changelist = self.lastP4Changelist()
2078                         self.modifyChangelistUser(changelist, p4User)
2079
2080                 submitted = True
2081
2082         finally:
2083             # skip this patch
2084             if not submitted or self.shelve:
2085                 if self.shelve:
2086                     print ("Reverting shelved files.")
2087                 else:
2088                     print ("Submission cancelled, undoing p4 changes.")
2089                 for f in editedFiles | filesToDelete:
2090                     p4_revert(f)
2091                 for f in filesToAdd:
2092                     p4_revert(f)
2093                     os.remove(f)
2094
2095         os.remove(fileName)
2096         return submitted
2097
2098     # Export git tags as p4 labels. Create a p4 label and then tag
2099     # with that.
2100     def exportGitTags(self, gitTags):
2101         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2102         if len(validLabelRegexp) == 0:
2103             validLabelRegexp = defaultLabelRegexp
2104         m = re.compile(validLabelRegexp)
2105
2106         for name in gitTags:
2107
2108             if not m.match(name):
2109                 if verbose:
2110                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2111                 continue
2112
2113             # Get the p4 commit this corresponds to
2114             logMessage = extractLogMessageFromGitCommit(name)
2115             values = extractSettingsGitLog(logMessage)
2116
2117             if 'change' not in values:
2118                 # a tag pointing to something not sent to p4; ignore
2119                 if verbose:
2120                     print("git tag %s does not give a p4 commit" % name)
2121                 continue
2122             else:
2123                 changelist = values['change']
2124
2125             # Get the tag details.
2126             inHeader = True
2127             isAnnotated = False
2128             body = []
2129             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2130                 l = l.strip()
2131                 if inHeader:
2132                     if re.match(r'tag\s+', l):
2133                         isAnnotated = True
2134                     elif re.match(r'\s*$', l):
2135                         inHeader = False
2136                         continue
2137                 else:
2138                     body.append(l)
2139
2140             if not isAnnotated:
2141                 body = ["lightweight tag imported by git p4\n"]
2142
2143             # Create the label - use the same view as the client spec we are using
2144             clientSpec = getClientSpec()
2145
2146             labelTemplate  = "Label: %s\n" % name
2147             labelTemplate += "Description:\n"
2148             for b in body:
2149                 labelTemplate += "\t" + b + "\n"
2150             labelTemplate += "View:\n"
2151             for depot_side in clientSpec.mappings:
2152                 labelTemplate += "\t%s\n" % depot_side
2153
2154             if self.dry_run:
2155                 print("Would create p4 label %s for tag" % name)
2156             elif self.prepare_p4_only:
2157                 print("Not creating p4 label %s for tag due to option" \
2158                       " --prepare-p4-only" % name)
2159             else:
2160                 p4_write_pipe(["label", "-i"], labelTemplate)
2161
2162                 # Use the label
2163                 p4_system(["tag", "-l", name] +
2164                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2165
2166                 if verbose:
2167                     print("created p4 label for tag %s" % name)
2168
2169     def run(self, args):
2170         if len(args) == 0:
2171             self.master = currentGitBranch()
2172         elif len(args) == 1:
2173             self.master = args[0]
2174             if not branchExists(self.master):
2175                 die("Branch %s does not exist" % self.master)
2176         else:
2177             return False
2178
2179         for i in self.update_shelve:
2180             if i <= 0:
2181                 sys.exit("invalid changelist %d" % i)
2182
2183         if self.master:
2184             allowSubmit = gitConfig("git-p4.allowSubmit")
2185             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2186                 die("%s is not in git-p4.allowSubmit" % self.master)
2187
2188         [upstream, settings] = findUpstreamBranchPoint()
2189         self.depotPath = settings['depot-paths'][0]
2190         if len(self.origin) == 0:
2191             self.origin = upstream
2192
2193         if len(self.update_shelve) > 0:
2194             self.shelve = True
2195
2196         if self.preserveUser:
2197             if not self.canChangeChangelists():
2198                 die("Cannot preserve user names without p4 super-user or admin permissions")
2199
2200         # if not set from the command line, try the config file
2201         if self.conflict_behavior is None:
2202             val = gitConfig("git-p4.conflict")
2203             if val:
2204                 if val not in self.conflict_behavior_choices:
2205                     die("Invalid value '%s' for config git-p4.conflict" % val)
2206             else:
2207                 val = "ask"
2208             self.conflict_behavior = val
2209
2210         if self.verbose:
2211             print("Origin branch is " + self.origin)
2212
2213         if len(self.depotPath) == 0:
2214             print("Internal error: cannot locate perforce depot path from existing branches")
2215             sys.exit(128)
2216
2217         self.useClientSpec = False
2218         if gitConfigBool("git-p4.useclientspec"):
2219             self.useClientSpec = True
2220         if self.useClientSpec:
2221             self.clientSpecDirs = getClientSpec()
2222
2223         # Check for the existence of P4 branches
2224         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2225
2226         if self.useClientSpec and not branchesDetected:
2227             # all files are relative to the client spec
2228             self.clientPath = getClientRoot()
2229         else:
2230             self.clientPath = p4Where(self.depotPath)
2231
2232         if self.clientPath == "":
2233             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2234
2235         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2236         self.oldWorkingDirectory = os.getcwd()
2237
2238         # ensure the clientPath exists
2239         new_client_dir = False
2240         if not os.path.exists(self.clientPath):
2241             new_client_dir = True
2242             os.makedirs(self.clientPath)
2243
2244         chdir(self.clientPath, is_client_path=True)
2245         if self.dry_run:
2246             print("Would synchronize p4 checkout in %s" % self.clientPath)
2247         else:
2248             print("Synchronizing p4 checkout...")
2249             if new_client_dir:
2250                 # old one was destroyed, and maybe nobody told p4
2251                 p4_sync("...", "-f")
2252             else:
2253                 p4_sync("...")
2254         self.check()
2255
2256         commits = []
2257         if self.master:
2258             committish = self.master
2259         else:
2260             committish = 'HEAD'
2261
2262         if self.commit != "":
2263             if self.commit.find("..") != -1:
2264                 limits_ish = self.commit.split("..")
2265                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2266                     commits.append(line.strip())
2267                 commits.reverse()
2268             else:
2269                 commits.append(self.commit)
2270         else:
2271             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2272                 commits.append(line.strip())
2273             commits.reverse()
2274
2275         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2276             self.checkAuthorship = False
2277         else:
2278             self.checkAuthorship = True
2279
2280         if self.preserveUser:
2281             self.checkValidP4Users(commits)
2282
2283         #
2284         # Build up a set of options to be passed to diff when
2285         # submitting each commit to p4.
2286         #
2287         if self.detectRenames:
2288             # command-line -M arg
2289             self.diffOpts = "-M"
2290         else:
2291             # If not explicitly set check the config variable
2292             detectRenames = gitConfig("git-p4.detectRenames")
2293
2294             if detectRenames.lower() == "false" or detectRenames == "":
2295                 self.diffOpts = ""
2296             elif detectRenames.lower() == "true":
2297                 self.diffOpts = "-M"
2298             else:
2299                 self.diffOpts = "-M%s" % detectRenames
2300
2301         # no command-line arg for -C or --find-copies-harder, just
2302         # config variables
2303         detectCopies = gitConfig("git-p4.detectCopies")
2304         if detectCopies.lower() == "false" or detectCopies == "":
2305             pass
2306         elif detectCopies.lower() == "true":
2307             self.diffOpts += " -C"
2308         else:
2309             self.diffOpts += " -C%s" % detectCopies
2310
2311         if gitConfigBool("git-p4.detectCopiesHarder"):
2312             self.diffOpts += " --find-copies-harder"
2313
2314         num_shelves = len(self.update_shelve)
2315         if num_shelves > 0 and num_shelves != len(commits):
2316             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2317                      (len(commits), num_shelves))
2318
2319         hooks_path = gitConfig("core.hooksPath")
2320         if len(hooks_path) <= 0:
2321             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2322
2323         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2324         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2325             sys.exit(1)
2326
2327         #
2328         # Apply the commits, one at a time.  On failure, ask if should
2329         # continue to try the rest of the patches, or quit.
2330         #
2331         if self.dry_run:
2332             print("Would apply")
2333         applied = []
2334         last = len(commits) - 1
2335         for i, commit in enumerate(commits):
2336             if self.dry_run:
2337                 print(" ", read_pipe(["git", "show", "-s",
2338                                       "--format=format:%h %s", commit]))
2339                 ok = True
2340             else:
2341                 ok = self.applyCommit(commit)
2342             if ok:
2343                 applied.append(commit)
2344             else:
2345                 if self.prepare_p4_only and i < last:
2346                     print("Processing only the first commit due to option" \
2347                           " --prepare-p4-only")
2348                     break
2349                 if i < last:
2350                     quit = False
2351                     while True:
2352                         # prompt for what to do, or use the option/variable
2353                         if self.conflict_behavior == "ask":
2354                             print("What do you want to do?")
2355                             response = raw_input("[s]kip this commit but apply"
2356                                                  " the rest, or [q]uit? ")
2357                             if not response:
2358                                 continue
2359                         elif self.conflict_behavior == "skip":
2360                             response = "s"
2361                         elif self.conflict_behavior == "quit":
2362                             response = "q"
2363                         else:
2364                             die("Unknown conflict_behavior '%s'" %
2365                                 self.conflict_behavior)
2366
2367                         if response[0] == "s":
2368                             print("Skipping this commit, but applying the rest")
2369                             break
2370                         if response[0] == "q":
2371                             print("Quitting")
2372                             quit = True
2373                             break
2374                     if quit:
2375                         break
2376
2377         chdir(self.oldWorkingDirectory)
2378         shelved_applied = "shelved" if self.shelve else "applied"
2379         if self.dry_run:
2380             pass
2381         elif self.prepare_p4_only:
2382             pass
2383         elif len(commits) == len(applied):
2384             print("All commits {0}!".format(shelved_applied))
2385
2386             sync = P4Sync()
2387             if self.branch:
2388                 sync.branch = self.branch
2389             if self.disable_p4sync:
2390                 sync.sync_origin_only()
2391             else:
2392                 sync.run([])
2393
2394                 if not self.disable_rebase:
2395                     rebase = P4Rebase()
2396                     rebase.rebase()
2397
2398         else:
2399             if len(applied) == 0:
2400                 print("No commits {0}.".format(shelved_applied))
2401             else:
2402                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2403                 for c in commits:
2404                     if c in applied:
2405                         star = "*"
2406                     else:
2407                         star = " "
2408                     print(star, read_pipe(["git", "show", "-s",
2409                                            "--format=format:%h %s",  c]))
2410                 print("You will have to do 'git p4 sync' and rebase.")
2411
2412         if gitConfigBool("git-p4.exportLabels"):
2413             self.exportLabels = True
2414
2415         if self.exportLabels:
2416             p4Labels = getP4Labels(self.depotPath)
2417             gitTags = getGitTags()
2418
2419             missingGitTags = gitTags - p4Labels
2420             self.exportGitTags(missingGitTags)
2421
2422         # exit with error unless everything applied perfectly
2423         if len(commits) != len(applied):
2424                 sys.exit(1)
2425
2426         return True
2427
2428 class View(object):
2429     """Represent a p4 view ("p4 help views"), and map files in a
2430        repo according to the view."""
2431
2432     def __init__(self, client_name):
2433         self.mappings = []
2434         self.client_prefix = "//%s/" % client_name
2435         # cache results of "p4 where" to lookup client file locations
2436         self.client_spec_path_cache = {}
2437
2438     def append(self, view_line):
2439         """Parse a view line, splitting it into depot and client
2440            sides.  Append to self.mappings, preserving order.  This
2441            is only needed for tag creation."""
2442
2443         # Split the view line into exactly two words.  P4 enforces
2444         # structure on these lines that simplifies this quite a bit.
2445         #
2446         # Either or both words may be double-quoted.
2447         # Single quotes do not matter.
2448         # Double-quote marks cannot occur inside the words.
2449         # A + or - prefix is also inside the quotes.
2450         # There are no quotes unless they contain a space.
2451         # The line is already white-space stripped.
2452         # The two words are separated by a single space.
2453         #
2454         if view_line[0] == '"':
2455             # First word is double quoted.  Find its end.
2456             close_quote_index = view_line.find('"', 1)
2457             if close_quote_index <= 0:
2458                 die("No first-word closing quote found: %s" % view_line)
2459             depot_side = view_line[1:close_quote_index]
2460             # skip closing quote and space
2461             rhs_index = close_quote_index + 1 + 1
2462         else:
2463             space_index = view_line.find(" ")
2464             if space_index <= 0:
2465                 die("No word-splitting space found: %s" % view_line)
2466             depot_side = view_line[0:space_index]
2467             rhs_index = space_index + 1
2468
2469         # prefix + means overlay on previous mapping
2470         if depot_side.startswith("+"):
2471             depot_side = depot_side[1:]
2472
2473         # prefix - means exclude this path, leave out of mappings
2474         exclude = False
2475         if depot_side.startswith("-"):
2476             exclude = True
2477             depot_side = depot_side[1:]
2478
2479         if not exclude:
2480             self.mappings.append(depot_side)
2481
2482     def convert_client_path(self, clientFile):
2483         # chop off //client/ part to make it relative
2484         if not clientFile.startswith(self.client_prefix):
2485             die("No prefix '%s' on clientFile '%s'" %
2486                 (self.client_prefix, clientFile))
2487         return clientFile[len(self.client_prefix):]
2488
2489     def update_client_spec_path_cache(self, files):
2490         """ Caching file paths by "p4 where" batch query """
2491
2492         # List depot file paths exclude that already cached
2493         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2494
2495         if len(fileArgs) == 0:
2496             return  # All files in cache
2497
2498         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2499         for res in where_result:
2500             if "code" in res and res["code"] == "error":
2501                 # assume error is "... file(s) not in client view"
2502                 continue
2503             if "clientFile" not in res:
2504                 die("No clientFile in 'p4 where' output")
2505             if "unmap" in res:
2506                 # it will list all of them, but only one not unmap-ped
2507                 continue
2508             if gitConfigBool("core.ignorecase"):
2509                 res['depotFile'] = res['depotFile'].lower()
2510             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2511
2512         # not found files or unmap files set to ""
2513         for depotFile in fileArgs:
2514             if gitConfigBool("core.ignorecase"):
2515                 depotFile = depotFile.lower()
2516             if depotFile not in self.client_spec_path_cache:
2517                 self.client_spec_path_cache[depotFile] = ""
2518
2519     def map_in_client(self, depot_path):
2520         """Return the relative location in the client where this
2521            depot file should live.  Returns "" if the file should
2522            not be mapped in the client."""
2523
2524         if gitConfigBool("core.ignorecase"):
2525             depot_path = depot_path.lower()
2526
2527         if depot_path in self.client_spec_path_cache:
2528             return self.client_spec_path_cache[depot_path]
2529
2530         die( "Error: %s is not found in client spec path" % depot_path )
2531         return ""
2532
2533 def cloneExcludeCallback(option, opt_str, value, parser):
2534     # prepend "/" because the first "/" was consumed as part of the option itself.
2535     # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2536     parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2537
2538 class P4Sync(Command, P4UserMap):
2539
2540     def __init__(self):
2541         Command.__init__(self)
2542         P4UserMap.__init__(self)
2543         self.options = [
2544                 optparse.make_option("--branch", dest="branch"),
2545                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2546                 optparse.make_option("--changesfile", dest="changesFile"),
2547                 optparse.make_option("--silent", dest="silent", action="store_true"),
2548                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2549                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2550                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2551                                      help="Import into refs/heads/ , not refs/remotes"),
2552                 optparse.make_option("--max-changes", dest="maxChanges",
2553                                      help="Maximum number of changes to import"),
2554                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2555                                      help="Internal block size to use when iteratively calling p4 changes"),
2556                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2557                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2558                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2559                                      help="Only sync files that are included in the Perforce Client Spec"),
2560                 optparse.make_option("-/", dest="cloneExclude",
2561                                      action="callback", callback=cloneExcludeCallback, type="string",
2562                                      help="exclude depot path"),
2563         ]
2564         self.description = """Imports from Perforce into a git repository.\n
2565     example:
2566     //depot/my/project/ -- to import the current head
2567     //depot/my/project/@all -- to import everything
2568     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2569
2570     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2571
2572         self.usage += " //depot/path[@revRange]"
2573         self.silent = False
2574         self.createdBranches = set()
2575         self.committedChanges = set()
2576         self.branch = ""
2577         self.detectBranches = False
2578         self.detectLabels = False
2579         self.importLabels = False
2580         self.changesFile = ""
2581         self.syncWithOrigin = True
2582         self.importIntoRemotes = True
2583         self.maxChanges = ""
2584         self.changes_block_size = None
2585         self.keepRepoPath = False
2586         self.depotPaths = None
2587         self.p4BranchesInGit = []
2588         self.cloneExclude = []
2589         self.useClientSpec = False
2590         self.useClientSpec_from_options = False
2591         self.clientSpecDirs = None
2592         self.tempBranches = []
2593         self.tempBranchLocation = "refs/git-p4-tmp"
2594         self.largeFileSystem = None
2595         self.suppress_meta_comment = False
2596
2597         if gitConfig('git-p4.largeFileSystem'):
2598             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2599             self.largeFileSystem = largeFileSystemConstructor(
2600                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2601             )
2602
2603         if gitConfig("git-p4.syncFromOrigin") == "false":
2604             self.syncWithOrigin = False
2605
2606         self.depotPaths = []
2607         self.changeRange = ""
2608         self.previousDepotPaths = []
2609         self.hasOrigin = False
2610
2611         # map from branch depot path to parent branch
2612         self.knownBranches = {}
2613         self.initialParents = {}
2614
2615         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2616         self.labels = {}
2617
2618     # Force a checkpoint in fast-import and wait for it to finish
2619     def checkpoint(self):
2620         self.gitStream.write("checkpoint\n\n")
2621         self.gitStream.write("progress checkpoint\n\n")
2622         out = self.gitOutput.readline()
2623         if self.verbose:
2624             print("checkpoint finished: " + out)
2625
2626     def isPathWanted(self, path):
2627         for p in self.cloneExclude:
2628             if p.endswith("/"):
2629                 if p4PathStartsWith(path, p):
2630                     return False
2631             # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2632             elif path.lower() == p.lower():
2633                 return False
2634         for p in self.depotPaths:
2635             if p4PathStartsWith(path, p):
2636                 return True
2637         return False
2638
2639     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2640         files = []
2641         fnum = 0
2642         while "depotFile%s" % fnum in commit:
2643             path =  commit["depotFile%s" % fnum]
2644             found = self.isPathWanted(path)
2645             if not found:
2646                 fnum = fnum + 1
2647                 continue
2648
2649             file = {}
2650             file["path"] = path
2651             file["rev"] = commit["rev%s" % fnum]
2652             file["action"] = commit["action%s" % fnum]
2653             file["type"] = commit["type%s" % fnum]
2654             if shelved:
2655                 file["shelved_cl"] = int(shelved_cl)
2656             files.append(file)
2657             fnum = fnum + 1
2658         return files
2659
2660     def extractJobsFromCommit(self, commit):
2661         jobs = []
2662         jnum = 0
2663         while "job%s" % jnum in commit:
2664             job = commit["job%s" % jnum]
2665             jobs.append(job)
2666             jnum = jnum + 1
2667         return jobs
2668
2669     def stripRepoPath(self, path, prefixes):
2670         """When streaming files, this is called to map a p4 depot path
2671            to where it should go in git.  The prefixes are either
2672            self.depotPaths, or self.branchPrefixes in the case of
2673            branch detection."""
2674
2675         if self.useClientSpec:
2676             # branch detection moves files up a level (the branch name)
2677             # from what client spec interpretation gives
2678             path = self.clientSpecDirs.map_in_client(path)
2679             if self.detectBranches:
2680                 for b in self.knownBranches:
2681                     if p4PathStartsWith(path, b + "/"):
2682                         path = path[len(b)+1:]
2683
2684         elif self.keepRepoPath:
2685             # Preserve everything in relative path name except leading
2686             # //depot/; just look at first prefix as they all should
2687             # be in the same depot.
2688             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2689             if p4PathStartsWith(path, depot):
2690                 path = path[len(depot):]
2691
2692         else:
2693             for p in prefixes:
2694                 if p4PathStartsWith(path, p):
2695                     path = path[len(p):]
2696                     break
2697
2698         path = wildcard_decode(path)
2699         return path
2700
2701     def splitFilesIntoBranches(self, commit):
2702         """Look at each depotFile in the commit to figure out to what
2703            branch it belongs."""
2704
2705         if self.clientSpecDirs:
2706             files = self.extractFilesFromCommit(commit)
2707             self.clientSpecDirs.update_client_spec_path_cache(files)
2708
2709         branches = {}
2710         fnum = 0
2711         while "depotFile%s" % fnum in commit:
2712             path =  commit["depotFile%s" % fnum]
2713             found = self.isPathWanted(path)
2714             if not found:
2715                 fnum = fnum + 1
2716                 continue
2717
2718             file = {}
2719             file["path"] = path
2720             file["rev"] = commit["rev%s" % fnum]
2721             file["action"] = commit["action%s" % fnum]
2722             file["type"] = commit["type%s" % fnum]
2723             fnum = fnum + 1
2724
2725             # start with the full relative path where this file would
2726             # go in a p4 client
2727             if self.useClientSpec:
2728                 relPath = self.clientSpecDirs.map_in_client(path)
2729             else:
2730                 relPath = self.stripRepoPath(path, self.depotPaths)
2731
2732             for branch in self.knownBranches.keys():
2733                 # add a trailing slash so that a commit into qt/4.2foo
2734                 # doesn't end up in qt/4.2, e.g.
2735                 if p4PathStartsWith(relPath, branch + "/"):
2736                     if branch not in branches:
2737                         branches[branch] = []
2738                     branches[branch].append(file)
2739                     break
2740
2741         return branches
2742
2743     def writeToGitStream(self, gitMode, relPath, contents):
2744         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2745         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2746         for d in contents:
2747             self.gitStream.write(d)
2748         self.gitStream.write('\n')
2749
2750     def encodeWithUTF8(self, path):
2751         try:
2752             path.decode('ascii')
2753         except:
2754             encoding = 'utf8'
2755             if gitConfig('git-p4.pathEncoding'):
2756                 encoding = gitConfig('git-p4.pathEncoding')
2757             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2758             if self.verbose:
2759                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2760         return path
2761
2762     # output one file from the P4 stream
2763     # - helper for streamP4Files
2764
2765     def streamOneP4File(self, file, contents):
2766         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2767         relPath = self.encodeWithUTF8(relPath)
2768         if verbose:
2769             if 'fileSize' in self.stream_file:
2770                 size = int(self.stream_file['fileSize'])
2771             else:
2772                 size = 0 # deleted files don't get a fileSize apparently
2773             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2774             sys.stdout.flush()
2775
2776         (type_base, type_mods) = split_p4_type(file["type"])
2777
2778         git_mode = "100644"
2779         if "x" in type_mods:
2780             git_mode = "100755"
2781         if type_base == "symlink":
2782             git_mode = "120000"
2783             # p4 print on a symlink sometimes contains "target\n";
2784             # if it does, remove the newline
2785             data = ''.join(contents)
2786             if not data:
2787                 # Some version of p4 allowed creating a symlink that pointed
2788                 # to nothing.  This causes p4 errors when checking out such
2789                 # a change, and errors here too.  Work around it by ignoring
2790                 # the bad symlink; hopefully a future change fixes it.
2791                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2792                 return
2793             elif data[-1] == '\n':
2794                 contents = [data[:-1]]
2795             else:
2796                 contents = [data]
2797
2798         if type_base == "utf16":
2799             # p4 delivers different text in the python output to -G
2800             # than it does when using "print -o", or normal p4 client
2801             # operations.  utf16 is converted to ascii or utf8, perhaps.
2802             # But ascii text saved as -t utf16 is completely mangled.
2803             # Invoke print -o to get the real contents.
2804             #
2805             # On windows, the newlines will always be mangled by print, so put
2806             # them back too.  This is not needed to the cygwin windows version,
2807             # just the native "NT" type.
2808             #
2809             try:
2810                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2811             except Exception as e:
2812                 if 'Translation of file content failed' in str(e):
2813                     type_base = 'binary'
2814                 else:
2815                     raise e
2816             else:
2817                 if p4_version_string().find('/NT') >= 0:
2818                     text = text.replace('\r\n', '\n')
2819                 contents = [ text ]
2820
2821         if type_base == "apple":
2822             # Apple filetype files will be streamed as a concatenation of
2823             # its appledouble header and the contents.  This is useless
2824             # on both macs and non-macs.  If using "print -q -o xx", it
2825             # will create "xx" with the data, and "%xx" with the header.
2826             # This is also not very useful.
2827             #
2828             # Ideally, someday, this script can learn how to generate
2829             # appledouble files directly and import those to git, but
2830             # non-mac machines can never find a use for apple filetype.
2831             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2832             return
2833
2834         # Note that we do not try to de-mangle keywords on utf16 files,
2835         # even though in theory somebody may want that.
2836         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2837         if pattern:
2838             regexp = re.compile(pattern, re.VERBOSE)
2839             text = ''.join(contents)
2840             text = regexp.sub(r'$\1$', text)
2841             contents = [ text ]
2842
2843         if self.largeFileSystem:
2844             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2845
2846         self.writeToGitStream(git_mode, relPath, contents)
2847
2848     def streamOneP4Deletion(self, file):
2849         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2850         relPath = self.encodeWithUTF8(relPath)
2851         if verbose:
2852             sys.stdout.write("delete %s\n" % relPath)
2853             sys.stdout.flush()
2854         self.gitStream.write("D %s\n" % relPath)
2855
2856         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2857             self.largeFileSystem.removeLargeFile(relPath)
2858
2859     # handle another chunk of streaming data
2860     def streamP4FilesCb(self, marshalled):
2861
2862         # catch p4 errors and complain
2863         err = None
2864         if "code" in marshalled:
2865             if marshalled["code"] == "error":
2866                 if "data" in marshalled:
2867                     err = marshalled["data"].rstrip()
2868
2869         if not err and 'fileSize' in self.stream_file:
2870             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2871             if required_bytes > 0:
2872                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2873                     os.getcwd(), required_bytes/1024/1024
2874                 )
2875
2876         if err:
2877             f = None
2878             if self.stream_have_file_info:
2879                 if "depotFile" in self.stream_file:
2880                     f = self.stream_file["depotFile"]
2881             # force a failure in fast-import, else an empty
2882             # commit will be made
2883             self.gitStream.write("\n")
2884             self.gitStream.write("die-now\n")
2885             self.gitStream.close()
2886             # ignore errors, but make sure it exits first
2887             self.importProcess.wait()
2888             if f:
2889                 die("Error from p4 print for %s: %s" % (f, err))
2890             else:
2891                 die("Error from p4 print: %s" % err)
2892
2893         if 'depotFile' in marshalled and self.stream_have_file_info:
2894             # start of a new file - output the old one first
2895             self.streamOneP4File(self.stream_file, self.stream_contents)
2896             self.stream_file = {}
2897             self.stream_contents = []
2898             self.stream_have_file_info = False
2899
2900         # pick up the new file information... for the
2901         # 'data' field we need to append to our array
2902         for k in marshalled.keys():
2903             if k == 'data':
2904                 if 'streamContentSize' not in self.stream_file:
2905                     self.stream_file['streamContentSize'] = 0
2906                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2907                 self.stream_contents.append(marshalled['data'])
2908             else:
2909                 self.stream_file[k] = marshalled[k]
2910
2911         if (verbose and
2912             'streamContentSize' in self.stream_file and
2913             'fileSize' in self.stream_file and
2914             'depotFile' in self.stream_file):
2915             size = int(self.stream_file["fileSize"])
2916             if size > 0:
2917                 progress = 100*self.stream_file['streamContentSize']/size
2918                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2919                 sys.stdout.flush()
2920
2921         self.stream_have_file_info = True
2922
2923     # Stream directly from "p4 files" into "git fast-import"
2924     def streamP4Files(self, files):
2925         filesForCommit = []
2926         filesToRead = []
2927         filesToDelete = []
2928
2929         for f in files:
2930             filesForCommit.append(f)
2931             if f['action'] in self.delete_actions:
2932                 filesToDelete.append(f)
2933             else:
2934                 filesToRead.append(f)
2935
2936         # deleted files...
2937         for f in filesToDelete:
2938             self.streamOneP4Deletion(f)
2939
2940         if len(filesToRead) > 0:
2941             self.stream_file = {}
2942             self.stream_contents = []
2943             self.stream_have_file_info = False
2944
2945             # curry self argument
2946             def streamP4FilesCbSelf(entry):
2947                 self.streamP4FilesCb(entry)
2948
2949             fileArgs = []
2950             for f in filesToRead:
2951                 if 'shelved_cl' in f:
2952                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2953                     # the contents
2954                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2955                 else:
2956                     fileArg = '%s#%s' % (f['path'], f['rev'])
2957
2958                 fileArgs.append(fileArg)
2959
2960             p4CmdList(["-x", "-", "print"],
2961                       stdin=fileArgs,
2962                       cb=streamP4FilesCbSelf)
2963
2964             # do the last chunk
2965             if 'depotFile' in self.stream_file:
2966                 self.streamOneP4File(self.stream_file, self.stream_contents)
2967
2968     def make_email(self, userid):
2969         if userid in self.users:
2970             return self.users[userid]
2971         else:
2972             return "%s <a@b>" % userid
2973
2974     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2975         """ Stream a p4 tag.
2976         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2977         """
2978
2979         if verbose:
2980             print("writing tag %s for commit %s" % (labelName, commit))
2981         gitStream.write("tag %s\n" % labelName)
2982         gitStream.write("from %s\n" % commit)
2983
2984         if 'Owner' in labelDetails:
2985             owner = labelDetails["Owner"]
2986         else:
2987             owner = None
2988
2989         # Try to use the owner of the p4 label, or failing that,
2990         # the current p4 user id.
2991         if owner:
2992             email = self.make_email(owner)
2993         else:
2994             email = self.make_email(self.p4UserId())
2995         tagger = "%s %s %s" % (email, epoch, self.tz)
2996
2997         gitStream.write("tagger %s\n" % tagger)
2998
2999         print("labelDetails=",labelDetails)
3000         if 'Description' in labelDetails:
3001             description = labelDetails['Description']
3002         else:
3003             description = 'Label from git p4'
3004
3005         gitStream.write("data %d\n" % len(description))
3006         gitStream.write(description)
3007         gitStream.write("\n")
3008
3009     def inClientSpec(self, path):
3010         if not self.clientSpecDirs:
3011             return True
3012         inClientSpec = self.clientSpecDirs.map_in_client(path)
3013         if not inClientSpec and self.verbose:
3014             print('Ignoring file outside of client spec: {0}'.format(path))
3015         return inClientSpec
3016
3017     def hasBranchPrefix(self, path):
3018         if not self.branchPrefixes:
3019             return True
3020         hasPrefix = [p for p in self.branchPrefixes
3021                         if p4PathStartsWith(path, p)]
3022         if not hasPrefix and self.verbose:
3023             print('Ignoring file outside of prefix: {0}'.format(path))
3024         return hasPrefix
3025
3026     def commit(self, details, files, branch, parent = "", allow_empty=False):
3027         epoch = details["time"]
3028         author = details["user"]
3029         jobs = self.extractJobsFromCommit(details)
3030
3031         if self.verbose:
3032             print('commit into {0}'.format(branch))
3033
3034         if self.clientSpecDirs:
3035             self.clientSpecDirs.update_client_spec_path_cache(files)
3036
3037         files = [f for f in files
3038             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3039
3040         if gitConfigBool('git-p4.keepEmptyCommits'):
3041             allow_empty = True
3042
3043         if not files and not allow_empty:
3044             print('Ignoring revision {0} as it would produce an empty commit.'
3045                 .format(details['change']))
3046             return
3047
3048         self.gitStream.write("commit %s\n" % branch)
3049         self.gitStream.write("mark :%s\n" % details["change"])
3050         self.committedChanges.add(int(details["change"]))
3051         committer = ""
3052         if author not in self.users:
3053             self.getUserMapFromPerforceServer()
3054         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3055
3056         self.gitStream.write("committer %s\n" % committer)
3057
3058         self.gitStream.write("data <<EOT\n")
3059         self.gitStream.write(details["desc"])
3060         if len(jobs) > 0:
3061             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3062
3063         if not self.suppress_meta_comment:
3064             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3065                                 (','.join(self.branchPrefixes), details["change"]))
3066             if len(details['options']) > 0:
3067                 self.gitStream.write(": options = %s" % details['options'])
3068             self.gitStream.write("]\n")
3069
3070         self.gitStream.write("EOT\n\n")
3071
3072         if len(parent) > 0:
3073             if self.verbose:
3074                 print("parent %s" % parent)
3075             self.gitStream.write("from %s\n" % parent)
3076
3077         self.streamP4Files(files)
3078         self.gitStream.write("\n")
3079
3080         change = int(details["change"])
3081
3082         if change in self.labels:
3083             label = self.labels[change]
3084             labelDetails = label[0]
3085             labelRevisions = label[1]
3086             if self.verbose:
3087                 print("Change %s is labelled %s" % (change, labelDetails))
3088
3089             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3090                                                 for p in self.branchPrefixes])
3091
3092             if len(files) == len(labelRevisions):
3093
3094                 cleanedFiles = {}
3095                 for info in files:
3096                     if info["action"] in self.delete_actions:
3097                         continue
3098                     cleanedFiles[info["depotFile"]] = info["rev"]
3099
3100                 if cleanedFiles == labelRevisions:
3101                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3102
3103                 else:
3104                     if not self.silent:
3105                         print("Tag %s does not match with change %s: files do not match."
3106                                % (labelDetails["label"], change))
3107
3108             else:
3109                 if not self.silent:
3110                     print("Tag %s does not match with change %s: file count is different."
3111                            % (labelDetails["label"], change))
3112
3113     # Build a dictionary of changelists and labels, for "detect-labels" option.
3114     def getLabels(self):
3115         self.labels = {}
3116
3117         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3118         if len(l) > 0 and not self.silent:
3119             print("Finding files belonging to labels in %s" % self.depotPaths)
3120
3121         for output in l:
3122             label = output["label"]
3123             revisions = {}
3124             newestChange = 0
3125             if self.verbose:
3126                 print("Querying files for label %s" % label)
3127             for file in p4CmdList(["files"] +
3128                                       ["%s...@%s" % (p, label)
3129                                           for p in self.depotPaths]):
3130                 revisions[file["depotFile"]] = file["rev"]
3131                 change = int(file["change"])
3132                 if change > newestChange:
3133                     newestChange = change
3134
3135             self.labels[newestChange] = [output, revisions]
3136
3137         if self.verbose:
3138             print("Label changes: %s" % self.labels.keys())
3139
3140     # Import p4 labels as git tags. A direct mapping does not
3141     # exist, so assume that if all the files are at the same revision
3142     # then we can use that, or it's something more complicated we should
3143     # just ignore.
3144     def importP4Labels(self, stream, p4Labels):
3145         if verbose:
3146             print("import p4 labels: " + ' '.join(p4Labels))
3147
3148         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3149         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3150         if len(validLabelRegexp) == 0:
3151             validLabelRegexp = defaultLabelRegexp
3152         m = re.compile(validLabelRegexp)
3153
3154         for name in p4Labels:
3155             commitFound = False
3156
3157             if not m.match(name):
3158                 if verbose:
3159                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3160                 continue
3161
3162             if name in ignoredP4Labels:
3163                 continue
3164
3165             labelDetails = p4CmdList(['label', "-o", name])[0]
3166
3167             # get the most recent changelist for each file in this label
3168             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3169                                 for p in self.depotPaths])
3170
3171             if 'change' in change:
3172                 # find the corresponding git commit; take the oldest commit
3173                 changelist = int(change['change'])
3174                 if changelist in self.committedChanges:
3175                     gitCommit = ":%d" % changelist       # use a fast-import mark
3176                     commitFound = True
3177                 else:
3178                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3179                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3180                     if len(gitCommit) == 0:
3181                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3182                     else:
3183                         commitFound = True
3184                         gitCommit = gitCommit.strip()
3185
3186                 if commitFound:
3187                     # Convert from p4 time format
3188                     try:
3189                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3190                     except ValueError:
3191                         print("Could not convert label time %s" % labelDetails['Update'])
3192                         tmwhen = 1
3193
3194                     when = int(time.mktime(tmwhen))
3195                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3196                     if verbose:
3197                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3198             else:
3199                 if verbose:
3200                     print("Label %s has no changelists - possibly deleted?" % name)
3201
3202             if not commitFound:
3203                 # We can't import this label; don't try again as it will get very
3204                 # expensive repeatedly fetching all the files for labels that will
3205                 # never be imported. If the label is moved in the future, the
3206                 # ignore will need to be removed manually.
3207                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3208
3209     def guessProjectName(self):
3210         for p in self.depotPaths:
3211             if p.endswith("/"):
3212                 p = p[:-1]
3213             p = p[p.strip().rfind("/") + 1:]
3214             if not p.endswith("/"):
3215                p += "/"
3216             return p
3217
3218     def getBranchMapping(self):
3219         lostAndFoundBranches = set()
3220
3221         user = gitConfig("git-p4.branchUser")
3222         if len(user) > 0:
3223             command = "branches -u %s" % user
3224         else:
3225             command = "branches"
3226
3227         for info in p4CmdList(command):
3228             details = p4Cmd(["branch", "-o", info["branch"]])
3229             viewIdx = 0
3230             while "View%s" % viewIdx in details:
3231                 paths = details["View%s" % viewIdx].split(" ")
3232                 viewIdx = viewIdx + 1
3233                 # require standard //depot/foo/... //depot/bar/... mapping
3234                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3235                     continue
3236                 source = paths[0]
3237                 destination = paths[1]
3238                 ## HACK
3239                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3240                     source = source[len(self.depotPaths[0]):-4]
3241                     destination = destination[len(self.depotPaths[0]):-4]
3242
3243                     if destination in self.knownBranches:
3244                         if not self.silent:
3245                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3246                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3247                         continue
3248
3249                     self.knownBranches[destination] = source
3250
3251                     lostAndFoundBranches.discard(destination)
3252
3253                     if source not in self.knownBranches:
3254                         lostAndFoundBranches.add(source)
3255
3256         # Perforce does not strictly require branches to be defined, so we also
3257         # check git config for a branch list.
3258         #
3259         # Example of branch definition in git config file:
3260         # [git-p4]
3261         #   branchList=main:branchA
3262         #   branchList=main:branchB
3263         #   branchList=branchA:branchC
3264         configBranches = gitConfigList("git-p4.branchList")
3265         for branch in configBranches:
3266             if branch:
3267                 (source, destination) = branch.split(":")
3268                 self.knownBranches[destination] = source
3269
3270                 lostAndFoundBranches.discard(destination)
3271
3272                 if source not in self.knownBranches:
3273                     lostAndFoundBranches.add(source)
3274
3275
3276         for branch in lostAndFoundBranches:
3277             self.knownBranches[branch] = branch
3278
3279     def getBranchMappingFromGitBranches(self):
3280         branches = p4BranchesInGit(self.importIntoRemotes)
3281         for branch in branches.keys():
3282             if branch == "master":
3283                 branch = "main"
3284             else:
3285                 branch = branch[len(self.projectName):]
3286             self.knownBranches[branch] = branch
3287
3288     def updateOptionDict(self, d):
3289         option_keys = {}
3290         if self.keepRepoPath:
3291             option_keys['keepRepoPath'] = 1
3292
3293         d["options"] = ' '.join(sorted(option_keys.keys()))
3294
3295     def readOptions(self, d):
3296         self.keepRepoPath = ('options' in d
3297                              and ('keepRepoPath' in d['options']))
3298
3299     def gitRefForBranch(self, branch):
3300         if branch == "main":
3301             return self.refPrefix + "master"
3302
3303         if len(branch) <= 0:
3304             return branch
3305
3306         return self.refPrefix + self.projectName + branch
3307
3308     def gitCommitByP4Change(self, ref, change):
3309         if self.verbose:
3310             print("looking in ref " + ref + " for change %s using bisect..." % change)
3311
3312         earliestCommit = ""
3313         latestCommit = parseRevision(ref)
3314
3315         while True:
3316             if self.verbose:
3317                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3318             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3319             if len(next) == 0:
3320                 if self.verbose:
3321                     print("argh")
3322                 return ""
3323             log = extractLogMessageFromGitCommit(next)
3324             settings = extractSettingsGitLog(log)
3325             currentChange = int(settings['change'])
3326             if self.verbose:
3327                 print("current change %s" % currentChange)
3328
3329             if currentChange == change:
3330                 if self.verbose:
3331                     print("found %s" % next)
3332                 return next
3333
3334             if currentChange < change:
3335                 earliestCommit = "^%s" % next
3336             else:
3337                 if next == latestCommit:
3338                     die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3339                 latestCommit = "%s^@" % next
3340
3341         return ""
3342
3343     def importNewBranch(self, branch, maxChange):
3344         # make fast-import flush all changes to disk and update the refs using the checkpoint
3345         # command so that we can try to find the branch parent in the git history
3346         self.gitStream.write("checkpoint\n\n");
3347         self.gitStream.flush();
3348         branchPrefix = self.depotPaths[0] + branch + "/"
3349         range = "@1,%s" % maxChange
3350         #print "prefix" + branchPrefix
3351         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3352         if len(changes) <= 0:
3353             return False
3354         firstChange = changes[0]
3355         #print "first change in branch: %s" % firstChange
3356         sourceBranch = self.knownBranches[branch]
3357         sourceDepotPath = self.depotPaths[0] + sourceBranch
3358         sourceRef = self.gitRefForBranch(sourceBranch)
3359         #print "source " + sourceBranch
3360
3361         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3362         #print "branch parent: %s" % branchParentChange
3363         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3364         if len(gitParent) > 0:
3365             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3366             #print "parent git commit: %s" % gitParent
3367
3368         self.importChanges(changes)
3369         return True
3370
3371     def searchParent(self, parent, branch, target):
3372         parentFound = False
3373         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3374                                      "--no-merges", parent]):
3375             blob = blob.strip()
3376             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3377                 parentFound = True
3378                 if self.verbose:
3379                     print("Found parent of %s in commit %s" % (branch, blob))
3380                 break
3381         if parentFound:
3382             return blob
3383         else:
3384             return None
3385
3386     def importChanges(self, changes, origin_revision=0):
3387         cnt = 1
3388         for change in changes:
3389             description = p4_describe(change)
3390             self.updateOptionDict(description)
3391
3392             if not self.silent:
3393                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3394                 sys.stdout.flush()
3395             cnt = cnt + 1
3396
3397             try:
3398                 if self.detectBranches:
3399                     branches = self.splitFilesIntoBranches(description)
3400                     for branch in branches.keys():
3401                         ## HACK  --hwn
3402                         branchPrefix = self.depotPaths[0] + branch + "/"
3403                         self.branchPrefixes = [ branchPrefix ]
3404
3405                         parent = ""
3406
3407                         filesForCommit = branches[branch]
3408
3409                         if self.verbose:
3410                             print("branch is %s" % branch)
3411
3412                         self.updatedBranches.add(branch)
3413
3414                         if branch not in self.createdBranches:
3415                             self.createdBranches.add(branch)
3416                             parent = self.knownBranches[branch]
3417                             if parent == branch:
3418                                 parent = ""
3419                             else:
3420                                 fullBranch = self.projectName + branch
3421                                 if fullBranch not in self.p4BranchesInGit:
3422                                     if not self.silent:
3423                                         print("\n    Importing new branch %s" % fullBranch);
3424                                     if self.importNewBranch(branch, change - 1):
3425                                         parent = ""
3426                                         self.p4BranchesInGit.append(fullBranch)
3427                                     if not self.silent:
3428                                         print("\n    Resuming with change %s" % change);
3429
3430                                 if self.verbose:
3431                                     print("parent determined through known branches: %s" % parent)
3432
3433                         branch = self.gitRefForBranch(branch)
3434                         parent = self.gitRefForBranch(parent)
3435
3436                         if self.verbose:
3437                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3438
3439                         if len(parent) == 0 and branch in self.initialParents:
3440                             parent = self.initialParents[branch]
3441                             del self.initialParents[branch]
3442
3443                         blob = None
3444                         if len(parent) > 0:
3445                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3446                             if self.verbose:
3447                                 print("Creating temporary branch: " + tempBranch)
3448                             self.commit(description, filesForCommit, tempBranch)
3449                             self.tempBranches.append(tempBranch)
3450                             self.checkpoint()
3451                             blob = self.searchParent(parent, branch, tempBranch)
3452                         if blob:
3453                             self.commit(description, filesForCommit, branch, blob)
3454                         else:
3455                             if self.verbose:
3456                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3457                             self.commit(description, filesForCommit, branch, parent)
3458                 else:
3459                     files = self.extractFilesFromCommit(description)
3460                     self.commit(description, files, self.branch,
3461                                 self.initialParent)
3462                     # only needed once, to connect to the previous commit
3463                     self.initialParent = ""
3464             except IOError:
3465                 print(self.gitError.read())
3466                 sys.exit(1)
3467
3468     def sync_origin_only(self):
3469         if self.syncWithOrigin:
3470             self.hasOrigin = originP4BranchesExist()
3471             if self.hasOrigin:
3472                 if not self.silent:
3473                     print('Syncing with origin first, using "git fetch origin"')
3474                 system("git fetch origin")
3475
3476     def importHeadRevision(self, revision):
3477         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3478
3479         details = {}
3480         details["user"] = "git perforce import user"
3481         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3482                            % (' '.join(self.depotPaths), revision))
3483         details["change"] = revision
3484         newestRevision = 0
3485
3486         fileCnt = 0
3487         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3488
3489         for info in p4CmdList(["files"] + fileArgs):
3490
3491             if 'code' in info and info['code'] == 'error':
3492                 sys.stderr.write("p4 returned an error: %s\n"
3493                                  % info['data'])
3494                 if info['data'].find("must refer to client") >= 0:
3495                     sys.stderr.write("This particular p4 error is misleading.\n")
3496                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3497                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3498                 sys.exit(1)
3499             if 'p4ExitCode' in info:
3500                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3501                 sys.exit(1)
3502
3503
3504             change = int(info["change"])
3505             if change > newestRevision:
3506                 newestRevision = change
3507
3508             if info["action"] in self.delete_actions:
3509                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3510                 #fileCnt = fileCnt + 1
3511                 continue
3512
3513             for prop in ["depotFile", "rev", "action", "type" ]:
3514                 details["%s%s" % (prop, fileCnt)] = info[prop]
3515
3516             fileCnt = fileCnt + 1
3517
3518         details["change"] = newestRevision
3519
3520         # Use time from top-most change so that all git p4 clones of
3521         # the same p4 repo have the same commit SHA1s.
3522         res = p4_describe(newestRevision)
3523         details["time"] = res["time"]
3524
3525         self.updateOptionDict(details)
3526         try:
3527             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3528         except IOError:
3529             print("IO error with git fast-import. Is your git version recent enough?")
3530             print(self.gitError.read())
3531
3532     def openStreams(self):
3533         self.importProcess = subprocess.Popen(["git", "fast-import"],
3534                                               stdin=subprocess.PIPE,
3535                                               stdout=subprocess.PIPE,
3536                                               stderr=subprocess.PIPE);
3537         self.gitOutput = self.importProcess.stdout
3538         self.gitStream = self.importProcess.stdin
3539         self.gitError = self.importProcess.stderr
3540
3541     def closeStreams(self):
3542         self.gitStream.close()
3543         if self.importProcess.wait() != 0:
3544             die("fast-import failed: %s" % self.gitError.read())
3545         self.gitOutput.close()
3546         self.gitError.close()
3547
3548     def run(self, args):
3549         if self.importIntoRemotes:
3550             self.refPrefix = "refs/remotes/p4/"
3551         else:
3552             self.refPrefix = "refs/heads/p4/"
3553
3554         self.sync_origin_only()
3555
3556         branch_arg_given = bool(self.branch)
3557         if len(self.branch) == 0:
3558             self.branch = self.refPrefix + "master"
3559             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3560                 system("git update-ref %s refs/heads/p4" % self.branch)
3561                 system("git branch -D p4")
3562
3563         # accept either the command-line option, or the configuration variable
3564         if self.useClientSpec:
3565             # will use this after clone to set the variable
3566             self.useClientSpec_from_options = True
3567         else:
3568             if gitConfigBool("git-p4.useclientspec"):
3569                 self.useClientSpec = True
3570         if self.useClientSpec:
3571             self.clientSpecDirs = getClientSpec()
3572
3573         # TODO: should always look at previous commits,
3574         # merge with previous imports, if possible.
3575         if args == []:
3576             if self.hasOrigin:
3577                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3578
3579             # branches holds mapping from branch name to sha1
3580             branches = p4BranchesInGit(self.importIntoRemotes)
3581
3582             # restrict to just this one, disabling detect-branches
3583             if branch_arg_given:
3584                 short = self.branch.split("/")[-1]
3585                 if short in branches:
3586                     self.p4BranchesInGit = [ short ]
3587             else:
3588                 self.p4BranchesInGit = branches.keys()
3589
3590             if len(self.p4BranchesInGit) > 1:
3591                 if not self.silent:
3592                     print("Importing from/into multiple branches")
3593                 self.detectBranches = True
3594                 for branch in branches.keys():
3595                     self.initialParents[self.refPrefix + branch] = \
3596                         branches[branch]
3597
3598             if self.verbose:
3599                 print("branches: %s" % self.p4BranchesInGit)
3600
3601             p4Change = 0
3602             for branch in self.p4BranchesInGit:
3603                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3604
3605                 settings = extractSettingsGitLog(logMsg)
3606
3607                 self.readOptions(settings)
3608                 if ('depot-paths' in settings
3609                     and 'change' in settings):
3610                     change = int(settings['change']) + 1
3611                     p4Change = max(p4Change, change)
3612
3613                     depotPaths = sorted(settings['depot-paths'])
3614                     if self.previousDepotPaths == []:
3615                         self.previousDepotPaths = depotPaths
3616                     else:
3617                         paths = []
3618                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3619                             prev_list = prev.split("/")
3620                             cur_list = cur.split("/")
3621                             for i in range(0, min(len(cur_list), len(prev_list))):
3622                                 if cur_list[i] != prev_list[i]:
3623                                     i = i - 1
3624                                     break
3625
3626                             paths.append ("/".join(cur_list[:i + 1]))
3627
3628                         self.previousDepotPaths = paths
3629
3630             if p4Change > 0:
3631                 self.depotPaths = sorted(self.previousDepotPaths)
3632                 self.changeRange = "@%s,#head" % p4Change
3633                 if not self.silent and not self.detectBranches:
3634                     print("Performing incremental import into %s git branch" % self.branch)
3635
3636         # accept multiple ref name abbreviations:
3637         #    refs/foo/bar/branch -> use it exactly
3638         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3639         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3640         if not self.branch.startswith("refs/"):
3641             if self.importIntoRemotes:
3642                 prepend = "refs/remotes/"
3643             else:
3644                 prepend = "refs/heads/"
3645             if not self.branch.startswith("p4/"):
3646                 prepend += "p4/"
3647             self.branch = prepend + self.branch
3648
3649         if len(args) == 0 and self.depotPaths:
3650             if not self.silent:
3651                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3652         else:
3653             if self.depotPaths and self.depotPaths != args:
3654                 print("previous import used depot path %s and now %s was specified. "
3655                        "This doesn't work!" % (' '.join (self.depotPaths),
3656                                                ' '.join (args)))
3657                 sys.exit(1)
3658
3659             self.depotPaths = sorted(args)
3660
3661         revision = ""
3662         self.users = {}
3663
3664         # Make sure no revision specifiers are used when --changesfile
3665         # is specified.
3666         bad_changesfile = False
3667         if len(self.changesFile) > 0:
3668             for p in self.depotPaths:
3669                 if p.find("@") >= 0 or p.find("#") >= 0:
3670                     bad_changesfile = True
3671                     break
3672         if bad_changesfile:
3673             die("Option --changesfile is incompatible with revision specifiers")
3674
3675         newPaths = []
3676         for p in self.depotPaths:
3677             if p.find("@") != -1:
3678                 atIdx = p.index("@")
3679                 self.changeRange = p[atIdx:]
3680                 if self.changeRange == "@all":
3681                     self.changeRange = ""
3682                 elif ',' not in self.changeRange:
3683                     revision = self.changeRange
3684                     self.changeRange = ""
3685                 p = p[:atIdx]
3686             elif p.find("#") != -1:
3687                 hashIdx = p.index("#")
3688                 revision = p[hashIdx:]
3689                 p = p[:hashIdx]
3690             elif self.previousDepotPaths == []:
3691                 # pay attention to changesfile, if given, else import
3692                 # the entire p4 tree at the head revision
3693                 if len(self.changesFile) == 0:
3694                     revision = "#head"
3695
3696             p = re.sub ("\.\.\.$", "", p)
3697             if not p.endswith("/"):
3698                 p += "/"
3699
3700             newPaths.append(p)
3701
3702         self.depotPaths = newPaths
3703
3704         # --detect-branches may change this for each branch
3705         self.branchPrefixes = self.depotPaths
3706
3707         self.loadUserMapFromCache()
3708         self.labels = {}
3709         if self.detectLabels:
3710             self.getLabels();
3711
3712         if self.detectBranches:
3713             ## FIXME - what's a P4 projectName ?
3714             self.projectName = self.guessProjectName()
3715
3716             if self.hasOrigin:
3717                 self.getBranchMappingFromGitBranches()
3718             else:
3719                 self.getBranchMapping()
3720             if self.verbose:
3721                 print("p4-git branches: %s" % self.p4BranchesInGit)
3722                 print("initial parents: %s" % self.initialParents)
3723             for b in self.p4BranchesInGit:
3724                 if b != "master":
3725
3726                     ## FIXME
3727                     b = b[len(self.projectName):]
3728                 self.createdBranches.add(b)
3729
3730         self.openStreams()
3731
3732         if revision:
3733             self.importHeadRevision(revision)
3734         else:
3735             changes = []
3736
3737             if len(self.changesFile) > 0:
3738                 output = open(self.changesFile).readlines()
3739                 changeSet = set()
3740                 for line in output:
3741                     changeSet.add(int(line))
3742
3743                 for change in changeSet:
3744                     changes.append(change)
3745
3746                 changes.sort()
3747             else:
3748                 # catch "git p4 sync" with no new branches, in a repo that
3749                 # does not have any existing p4 branches
3750                 if len(args) == 0:
3751                     if not self.p4BranchesInGit:
3752                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3753
3754                     # The default branch is master, unless --branch is used to
3755                     # specify something else.  Make sure it exists, or complain
3756                     # nicely about how to use --branch.
3757                     if not self.detectBranches:
3758                         if not branch_exists(self.branch):
3759                             if branch_arg_given:
3760                                 die("Error: branch %s does not exist." % self.branch)
3761                             else:
3762                                 die("Error: no branch %s; perhaps specify one with --branch." %
3763                                     self.branch)
3764
3765                 if self.verbose:
3766                     print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3767                                                               self.changeRange))
3768                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3769
3770                 if len(self.maxChanges) > 0:
3771                     changes = changes[:min(int(self.maxChanges), len(changes))]
3772
3773             if len(changes) == 0:
3774                 if not self.silent:
3775                     print("No changes to import!")
3776             else:
3777                 if not self.silent and not self.detectBranches:
3778                     print("Import destination: %s" % self.branch)
3779
3780                 self.updatedBranches = set()
3781
3782                 if not self.detectBranches:
3783                     if args:
3784                         # start a new branch
3785                         self.initialParent = ""
3786                     else:
3787                         # build on a previous revision
3788                         self.initialParent = parseRevision(self.branch)
3789
3790                 self.importChanges(changes)
3791
3792                 if not self.silent:
3793                     print("")
3794                     if len(self.updatedBranches) > 0:
3795                         sys.stdout.write("Updated branches: ")
3796                         for b in self.updatedBranches:
3797                             sys.stdout.write("%s " % b)
3798                         sys.stdout.write("\n")
3799
3800         if gitConfigBool("git-p4.importLabels"):
3801             self.importLabels = True
3802
3803         if self.importLabels:
3804             p4Labels = getP4Labels(self.depotPaths)
3805             gitTags = getGitTags()
3806
3807             missingP4Labels = p4Labels - gitTags
3808             self.importP4Labels(self.gitStream, missingP4Labels)
3809
3810         self.closeStreams()
3811
3812         # Cleanup temporary branches created during import
3813         if self.tempBranches != []:
3814             for branch in self.tempBranches:
3815                 read_pipe("git update-ref -d %s" % branch)
3816             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3817
3818         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3819         # a convenient shortcut refname "p4".
3820         if self.importIntoRemotes:
3821             head_ref = self.refPrefix + "HEAD"
3822             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3823                 system(["git", "symbolic-ref", head_ref, self.branch])
3824
3825         return True
3826
3827 class P4Rebase(Command):
3828     def __init__(self):
3829         Command.__init__(self)
3830         self.options = [
3831                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3832         ]
3833         self.importLabels = False
3834         self.description = ("Fetches the latest revision from perforce and "
3835                             + "rebases the current work (branch) against it")
3836
3837     def run(self, args):
3838         sync = P4Sync()
3839         sync.importLabels = self.importLabels
3840         sync.run([])
3841
3842         return self.rebase()
3843
3844     def rebase(self):
3845         if os.system("git update-index --refresh") != 0:
3846             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3847         if len(read_pipe("git diff-index HEAD --")) > 0:
3848             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3849
3850         [upstream, settings] = findUpstreamBranchPoint()
3851         if len(upstream) == 0:
3852             die("Cannot find upstream branchpoint for rebase")
3853
3854         # the branchpoint may be p4/foo~3, so strip off the parent
3855         upstream = re.sub("~[0-9]+$", "", upstream)
3856
3857         print("Rebasing the current branch onto %s" % upstream)
3858         oldHead = read_pipe("git rev-parse HEAD").strip()
3859         system("git rebase %s" % upstream)
3860         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3861         return True
3862
3863 class P4Clone(P4Sync):
3864     def __init__(self):
3865         P4Sync.__init__(self)
3866         self.description = "Creates a new git repository and imports from Perforce into it"
3867         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3868         self.options += [
3869             optparse.make_option("--destination", dest="cloneDestination",
3870                                  action='store', default=None,
3871                                  help="where to leave result of the clone"),
3872             optparse.make_option("--bare", dest="cloneBare",
3873                                  action="store_true", default=False),
3874         ]
3875         self.cloneDestination = None
3876         self.needsGit = False
3877         self.cloneBare = False
3878
3879     def defaultDestination(self, args):
3880         ## TODO: use common prefix of args?
3881         depotPath = args[0]
3882         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3883         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3884         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3885         depotDir = re.sub(r"/$", "", depotDir)
3886         return os.path.split(depotDir)[1]
3887
3888     def run(self, args):
3889         if len(args) < 1:
3890             return False
3891
3892         if self.keepRepoPath and not self.cloneDestination:
3893             sys.stderr.write("Must specify destination for --keep-path\n")
3894             sys.exit(1)
3895
3896         depotPaths = args
3897
3898         if not self.cloneDestination and len(depotPaths) > 1:
3899             self.cloneDestination = depotPaths[-1]
3900             depotPaths = depotPaths[:-1]
3901
3902         for p in depotPaths:
3903             if not p.startswith("//"):
3904                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3905                 return False
3906
3907         if not self.cloneDestination:
3908             self.cloneDestination = self.defaultDestination(args)
3909
3910         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3911
3912         if not os.path.exists(self.cloneDestination):
3913             os.makedirs(self.cloneDestination)
3914         chdir(self.cloneDestination)
3915
3916         init_cmd = [ "git", "init" ]
3917         if self.cloneBare:
3918             init_cmd.append("--bare")
3919         retcode = subprocess.call(init_cmd)
3920         if retcode:
3921             raise CalledProcessError(retcode, init_cmd)
3922
3923         if not P4Sync.run(self, depotPaths):
3924             return False
3925
3926         # create a master branch and check out a work tree
3927         if gitBranchExists(self.branch):
3928             system([ "git", "branch", "master", self.branch ])
3929             if not self.cloneBare:
3930                 system([ "git", "checkout", "-f" ])
3931         else:
3932             print('Not checking out any branch, use ' \
3933                   '"git checkout -q -b master <branch>"')
3934
3935         # auto-set this variable if invoked with --use-client-spec
3936         if self.useClientSpec_from_options:
3937             system("git config --bool git-p4.useclientspec true")
3938
3939         return True
3940
3941 class P4Unshelve(Command):
3942     def __init__(self):
3943         Command.__init__(self)
3944         self.options = []
3945         self.origin = "HEAD"
3946         self.description = "Unshelve a P4 changelist into a git commit"
3947         self.usage = "usage: %prog [options] changelist"
3948         self.options += [
3949                 optparse.make_option("--origin", dest="origin",
3950                     help="Use this base revision instead of the default (%s)" % self.origin),
3951         ]
3952         self.verbose = False
3953         self.noCommit = False
3954         self.destbranch = "refs/remotes/p4-unshelved"
3955
3956     def renameBranch(self, branch_name):
3957         """ Rename the existing branch to branch_name.N
3958         """
3959
3960         found = True
3961         for i in range(0,1000):
3962             backup_branch_name = "{0}.{1}".format(branch_name, i)
3963             if not gitBranchExists(backup_branch_name):
3964                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3965                 gitDeleteRef(branch_name)
3966                 found = True
3967                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3968                 break
3969
3970         if not found:
3971             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3972
3973     def findLastP4Revision(self, starting_point):
3974         """ Look back from starting_point for the first commit created by git-p4
3975             to find the P4 commit we are based on, and the depot-paths.
3976         """
3977
3978         for parent in (range(65535)):
3979             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3980             settings = extractSettingsGitLog(log)
3981             if 'change' in settings:
3982                 return settings
3983
3984         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3985
3986     def createShelveParent(self, change, branch_name, sync, origin):
3987         """ Create a commit matching the parent of the shelved changelist 'change'
3988         """
3989         parent_description = p4_describe(change, shelved=True)
3990         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
3991         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
3992
3993         parent_files = []
3994         for f in files:
3995             # if it was added in the shelved changelist, it won't exist in the parent
3996             if f['action'] in self.add_actions:
3997                 continue
3998
3999             # if it was deleted in the shelved changelist it must not be deleted
4000             # in the parent - we might even need to create it if the origin branch
4001             # does not have it
4002             if f['action'] in self.delete_actions:
4003                 f['action'] = 'add'
4004
4005             parent_files.append(f)
4006
4007         sync.commit(parent_description, parent_files, branch_name,
4008                 parent=origin, allow_empty=True)
4009         print("created parent commit for {0} based on {1} in {2}".format(
4010             change, self.origin, branch_name))
4011
4012     def run(self, args):
4013         if len(args) != 1:
4014             return False
4015
4016         if not gitBranchExists(self.origin):
4017             sys.exit("origin branch {0} does not exist".format(self.origin))
4018
4019         sync = P4Sync()
4020         changes = args
4021
4022         # only one change at a time
4023         change = changes[0]
4024
4025         # if the target branch already exists, rename it
4026         branch_name = "{0}/{1}".format(self.destbranch, change)
4027         if gitBranchExists(branch_name):
4028             self.renameBranch(branch_name)
4029         sync.branch = branch_name
4030
4031         sync.verbose = self.verbose
4032         sync.suppress_meta_comment = True
4033
4034         settings = self.findLastP4Revision(self.origin)
4035         sync.depotPaths = settings['depot-paths']
4036         sync.branchPrefixes = sync.depotPaths
4037
4038         sync.openStreams()
4039         sync.loadUserMapFromCache()
4040         sync.silent = True
4041
4042         # create a commit for the parent of the shelved changelist
4043         self.createShelveParent(change, branch_name, sync, self.origin)
4044
4045         # create the commit for the shelved changelist itself
4046         description = p4_describe(change, True)
4047         files = sync.extractFilesFromCommit(description, True, change)
4048
4049         sync.commit(description, files, branch_name, "")
4050         sync.closeStreams()
4051
4052         print("unshelved changelist {0} into {1}".format(change, branch_name))
4053
4054         return True
4055
4056 class P4Branches(Command):
4057     def __init__(self):
4058         Command.__init__(self)
4059         self.options = [ ]
4060         self.description = ("Shows the git branches that hold imports and their "
4061                             + "corresponding perforce depot paths")
4062         self.verbose = False
4063
4064     def run(self, args):
4065         if originP4BranchesExist():
4066             createOrUpdateBranchesFromOrigin()
4067
4068         cmdline = "git rev-parse --symbolic "
4069         cmdline += " --remotes"
4070
4071         for line in read_pipe_lines(cmdline):
4072             line = line.strip()
4073
4074             if not line.startswith('p4/') or line == "p4/HEAD":
4075                 continue
4076             branch = line
4077
4078             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4079             settings = extractSettingsGitLog(log)
4080
4081             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4082         return True
4083
4084 class HelpFormatter(optparse.IndentedHelpFormatter):
4085     def __init__(self):
4086         optparse.IndentedHelpFormatter.__init__(self)
4087
4088     def format_description(self, description):
4089         if description:
4090             return description + "\n"
4091         else:
4092             return ""
4093
4094 def printUsage(commands):
4095     print("usage: %s <command> [options]" % sys.argv[0])
4096     print("")
4097     print("valid commands: %s" % ", ".join(commands))
4098     print("")
4099     print("Try %s <command> --help for command specific help." % sys.argv[0])
4100     print("")
4101
4102 commands = {
4103     "debug" : P4Debug,
4104     "submit" : P4Submit,
4105     "commit" : P4Submit,
4106     "sync" : P4Sync,
4107     "rebase" : P4Rebase,
4108     "clone" : P4Clone,
4109     "rollback" : P4RollBack,
4110     "branches" : P4Branches,
4111     "unshelve" : P4Unshelve,
4112 }
4113
4114
4115 def main():
4116     if len(sys.argv[1:]) == 0:
4117         printUsage(commands.keys())
4118         sys.exit(2)
4119
4120     cmdName = sys.argv[1]
4121     try:
4122         klass = commands[cmdName]
4123         cmd = klass()
4124     except KeyError:
4125         print("unknown command %s" % cmdName)
4126         print("")
4127         printUsage(commands.keys())
4128         sys.exit(2)
4129
4130     options = cmd.options
4131     cmd.gitdir = os.environ.get("GIT_DIR", None)
4132
4133     args = sys.argv[2:]
4134
4135     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4136     if cmd.needsGit:
4137         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4138
4139     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4140                                    options,
4141                                    description = cmd.description,
4142                                    formatter = HelpFormatter())
4143
4144     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4145     global verbose
4146     verbose = cmd.verbose
4147     if cmd.needsGit:
4148         if cmd.gitdir == None:
4149             cmd.gitdir = os.path.abspath(".git")
4150             if not isValidGitDir(cmd.gitdir):
4151                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4152                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4153                 if os.path.exists(cmd.gitdir):
4154                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4155                     if len(cdup) > 0:
4156                         chdir(cdup);
4157
4158         if not isValidGitDir(cmd.gitdir):
4159             if isValidGitDir(cmd.gitdir + "/.git"):
4160                 cmd.gitdir += "/.git"
4161             else:
4162                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4163
4164         # so git commands invoked from the P4 workspace will succeed
4165         os.environ["GIT_DIR"] = cmd.gitdir
4166
4167     if not cmd.run(args):
4168         parser.print_help()
4169         sys.exit(2)
4170
4171
4172 if __name__ == '__main__':
4173     main()