Merge branch 'kn/userdiff-php'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 # support basestring in python3
31 try:
32     unicode = unicode
33 except NameError:
34     # 'unicode' is undefined, must be Python 3
35     str = str
36     unicode = str
37     bytes = bytes
38     basestring = (str,bytes)
39 else:
40     # 'unicode' exists, must be Python 2
41     str = str
42     unicode = unicode
43     bytes = str
44     basestring = basestring
45
46 try:
47     from subprocess import CalledProcessError
48 except ImportError:
49     # from python2.7:subprocess.py
50     # Exception classes used by this module.
51     class CalledProcessError(Exception):
52         """This exception is raised when a process run by check_call() returns
53         a non-zero exit status.  The exit status will be stored in the
54         returncode attribute."""
55         def __init__(self, returncode, cmd):
56             self.returncode = returncode
57             self.cmd = cmd
58         def __str__(self):
59             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
61 verbose = False
62
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
65
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
68
69 p4_access_checked = False
70
71 def p4_build_cmd(cmd):
72     """Build a suitable p4 command line.
73
74     This consolidates building and returning a p4 command line into one
75     location. It means that hooking into the environment, or other configuration
76     can be done more easily.
77     """
78     real_cmd = ["p4"]
79
80     user = gitConfig("git-p4.user")
81     if len(user) > 0:
82         real_cmd += ["-u",user]
83
84     password = gitConfig("git-p4.password")
85     if len(password) > 0:
86         real_cmd += ["-P", password]
87
88     port = gitConfig("git-p4.port")
89     if len(port) > 0:
90         real_cmd += ["-p", port]
91
92     host = gitConfig("git-p4.host")
93     if len(host) > 0:
94         real_cmd += ["-H", host]
95
96     client = gitConfig("git-p4.client")
97     if len(client) > 0:
98         real_cmd += ["-c", client]
99
100     retries = gitConfigInt("git-p4.retries")
101     if retries is None:
102         # Perform 3 retries by default
103         retries = 3
104     if retries > 0:
105         # Provide a way to not pass this option by setting git-p4.retries to 0
106         real_cmd += ["-r", str(retries)]
107
108     if isinstance(cmd,basestring):
109         real_cmd = ' '.join(real_cmd) + ' ' + cmd
110     else:
111         real_cmd += cmd
112
113     # now check that we can actually talk to the server
114     global p4_access_checked
115     if not p4_access_checked:
116         p4_access_checked = True    # suppress access checks in p4_check_access itself
117         p4_check_access()
118
119     return real_cmd
120
121 def git_dir(path):
122     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123         This won't automatically add ".git" to a directory.
124     """
125     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126     if not d or len(d) == 0:
127         return None
128     else:
129         return d
130
131 def chdir(path, is_client_path=False):
132     """Do chdir to the given path, and set the PWD environment
133        variable for use by P4.  It does not look at getcwd() output.
134        Since we're not using the shell, it is necessary to set the
135        PWD environment variable explicitly.
136
137        Normally, expand the path to force it to be absolute.  This
138        addresses the use of relative path names inside P4 settings,
139        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
140        as given; it looks for .p4config using PWD.
141
142        If is_client_path, the path was handed to us directly by p4,
143        and may be a symbolic link.  Do not call os.getcwd() in this
144        case, because it will cause p4 to think that PWD is not inside
145        the client path.
146        """
147
148     os.chdir(path)
149     if not is_client_path:
150         path = os.getcwd()
151     os.environ['PWD'] = path
152
153 def calcDiskFree():
154     """Return free space in bytes on the disk of the given dirname."""
155     if platform.system() == 'Windows':
156         free_bytes = ctypes.c_ulonglong(0)
157         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158         return free_bytes.value
159     else:
160         st = os.statvfs(os.getcwd())
161         return st.f_bavail * st.f_frsize
162
163 def die(msg):
164     if verbose:
165         raise Exception(msg)
166     else:
167         sys.stderr.write(msg + "\n")
168         sys.exit(1)
169
170 def write_pipe(c, stdin):
171     if verbose:
172         sys.stderr.write('Writing pipe: %s\n' % str(c))
173
174     expand = isinstance(c,basestring)
175     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176     pipe = p.stdin
177     val = pipe.write(stdin)
178     pipe.close()
179     if p.wait():
180         die('Command failed: %s' % str(c))
181
182     return val
183
184 def p4_write_pipe(c, stdin):
185     real_cmd = p4_build_cmd(c)
186     return write_pipe(real_cmd, stdin)
187
188 def read_pipe_full(c):
189     """ Read output from  command. Returns a tuple
190         of the return status, stdout text and stderr
191         text.
192     """
193     if verbose:
194         sys.stderr.write('Reading pipe: %s\n' % str(c))
195
196     expand = isinstance(c,basestring)
197     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198     (out, err) = p.communicate()
199     return (p.returncode, out, err)
200
201 def read_pipe(c, ignore_error=False):
202     """ Read output from  command. Returns the output text on
203         success. On failure, terminates execution, unless
204         ignore_error is True, when it returns an empty string.
205     """
206     (retcode, out, err) = read_pipe_full(c)
207     if retcode != 0:
208         if ignore_error:
209             out = ""
210         else:
211             die('Command failed: %s\nError: %s' % (str(c), err))
212     return out
213
214 def read_pipe_text(c):
215     """ Read output from a command with trailing whitespace stripped.
216         On error, returns None.
217     """
218     (retcode, out, err) = read_pipe_full(c)
219     if retcode != 0:
220         return None
221     else:
222         return out.rstrip()
223
224 def p4_read_pipe(c, ignore_error=False):
225     real_cmd = p4_build_cmd(c)
226     return read_pipe(real_cmd, ignore_error)
227
228 def read_pipe_lines(c):
229     if verbose:
230         sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232     expand = isinstance(c, basestring)
233     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234     pipe = p.stdout
235     val = pipe.readlines()
236     if pipe.close() or p.wait():
237         die('Command failed: %s' % str(c))
238
239     return val
240
241 def p4_read_pipe_lines(c):
242     """Specifically invoke p4 on the command supplied. """
243     real_cmd = p4_build_cmd(c)
244     return read_pipe_lines(real_cmd)
245
246 def p4_has_command(cmd):
247     """Ask p4 for help on this command.  If it returns an error, the
248        command does not exist in this version of p4."""
249     real_cmd = p4_build_cmd(["help", cmd])
250     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251                                    stderr=subprocess.PIPE)
252     p.communicate()
253     return p.returncode == 0
254
255 def p4_has_move_command():
256     """See if the move command exists, that it supports -k, and that
257        it has not been administratively disabled.  The arguments
258        must be correct, but the filenames do not have to exist.  Use
259        ones with wildcards so even if they exist, it will fail."""
260
261     if not p4_has_command("move"):
262         return False
263     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265     (out, err) = p.communicate()
266     # return code will be 1 in either case
267     if err.find("Invalid option") >= 0:
268         return False
269     if err.find("disabled") >= 0:
270         return False
271     # assume it failed because @... was invalid changelist
272     return True
273
274 def system(cmd, ignore_error=False):
275     expand = isinstance(cmd,basestring)
276     if verbose:
277         sys.stderr.write("executing %s\n" % str(cmd))
278     retcode = subprocess.call(cmd, shell=expand)
279     if retcode and not ignore_error:
280         raise CalledProcessError(retcode, cmd)
281
282     return retcode
283
284 def p4_system(cmd):
285     """Specifically invoke p4 as the system command. """
286     real_cmd = p4_build_cmd(cmd)
287     expand = isinstance(real_cmd, basestring)
288     retcode = subprocess.call(real_cmd, shell=expand)
289     if retcode:
290         raise CalledProcessError(retcode, real_cmd)
291
292 def die_bad_access(s):
293     die("failure accessing depot: {0}".format(s.rstrip()))
294
295 def p4_check_access(min_expiration=1):
296     """ Check if we can access Perforce - account still logged in
297     """
298     results = p4CmdList(["login", "-s"])
299
300     if len(results) == 0:
301         # should never get here: always get either some results, or a p4ExitCode
302         assert("could not parse response from perforce")
303
304     result = results[0]
305
306     if 'p4ExitCode' in result:
307         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308         die_bad_access("could not run p4")
309
310     code = result.get("code")
311     if not code:
312         # we get here if we couldn't connect and there was nothing to unmarshal
313         die_bad_access("could not connect")
314
315     elif code == "stat":
316         expiry = result.get("TicketExpiration")
317         if expiry:
318             expiry = int(expiry)
319             if expiry > min_expiration:
320                 # ok to carry on
321                 return
322             else:
323                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325         else:
326             # account without a timeout - all ok
327             return
328
329     elif code == "error":
330         data = result.get("data")
331         if data:
332             die_bad_access("p4 error: {0}".format(data))
333         else:
334             die_bad_access("unknown error")
335     else:
336         die_bad_access("unknown error code {0}".format(code))
337
338 _p4_version_string = None
339 def p4_version_string():
340     """Read the version string, showing just the last line, which
341        hopefully is the interesting version bit.
342
343        $ p4 -V
344        Perforce - The Fast Software Configuration Management System.
345        Copyright 1995-2011 Perforce Software.  All rights reserved.
346        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
347     """
348     global _p4_version_string
349     if not _p4_version_string:
350         a = p4_read_pipe_lines(["-V"])
351         _p4_version_string = a[-1].rstrip()
352     return _p4_version_string
353
354 def p4_integrate(src, dest):
355     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
356
357 def p4_sync(f, *options):
358     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
359
360 def p4_add(f):
361     # forcibly add file names with wildcards
362     if wildcard_present(f):
363         p4_system(["add", "-f", f])
364     else:
365         p4_system(["add", f])
366
367 def p4_delete(f):
368     p4_system(["delete", wildcard_encode(f)])
369
370 def p4_edit(f, *options):
371     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
372
373 def p4_revert(f):
374     p4_system(["revert", wildcard_encode(f)])
375
376 def p4_reopen(type, f):
377     p4_system(["reopen", "-t", type, wildcard_encode(f)])
378
379 def p4_reopen_in_change(changelist, files):
380     cmd = ["reopen", "-c", str(changelist)] + files
381     p4_system(cmd)
382
383 def p4_move(src, dest):
384     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
385
386 def p4_last_change():
387     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
388     return int(results[0]['change'])
389
390 def p4_describe(change, shelved=False):
391     """Make sure it returns a valid result by checking for
392        the presence of field "time".  Return a dict of the
393        results."""
394
395     cmd = ["describe", "-s"]
396     if shelved:
397         cmd += ["-S"]
398     cmd += [str(change)]
399
400     ds = p4CmdList(cmd, skip_info=True)
401     if len(ds) != 1:
402         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
403
404     d = ds[0]
405
406     if "p4ExitCode" in d:
407         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
408                                                       str(d)))
409     if "code" in d:
410         if d["code"] == "error":
411             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
412
413     if "time" not in d:
414         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
415
416     return d
417
418 #
419 # Canonicalize the p4 type and return a tuple of the
420 # base type, plus any modifiers.  See "p4 help filetypes"
421 # for a list and explanation.
422 #
423 def split_p4_type(p4type):
424
425     p4_filetypes_historical = {
426         "ctempobj": "binary+Sw",
427         "ctext": "text+C",
428         "cxtext": "text+Cx",
429         "ktext": "text+k",
430         "kxtext": "text+kx",
431         "ltext": "text+F",
432         "tempobj": "binary+FSw",
433         "ubinary": "binary+F",
434         "uresource": "resource+F",
435         "uxbinary": "binary+Fx",
436         "xbinary": "binary+x",
437         "xltext": "text+Fx",
438         "xtempobj": "binary+Swx",
439         "xtext": "text+x",
440         "xunicode": "unicode+x",
441         "xutf16": "utf16+x",
442     }
443     if p4type in p4_filetypes_historical:
444         p4type = p4_filetypes_historical[p4type]
445     mods = ""
446     s = p4type.split("+")
447     base = s[0]
448     mods = ""
449     if len(s) > 1:
450         mods = s[1]
451     return (base, mods)
452
453 #
454 # return the raw p4 type of a file (text, text+ko, etc)
455 #
456 def p4_type(f):
457     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
458     return results[0]['headType']
459
460 #
461 # Given a type base and modifier, return a regexp matching
462 # the keywords that can be expanded in the file
463 #
464 def p4_keywords_regexp_for_type(base, type_mods):
465     if base in ("text", "unicode", "binary"):
466         kwords = None
467         if "ko" in type_mods:
468             kwords = 'Id|Header'
469         elif "k" in type_mods:
470             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
471         else:
472             return None
473         pattern = r"""
474             \$              # Starts with a dollar, followed by...
475             (%s)            # one of the keywords, followed by...
476             (:[^$\n]+)?     # possibly an old expansion, followed by...
477             \$              # another dollar
478             """ % kwords
479         return pattern
480     else:
481         return None
482
483 #
484 # Given a file, return a regexp matching the possible
485 # RCS keywords that will be expanded, or None for files
486 # with kw expansion turned off.
487 #
488 def p4_keywords_regexp_for_file(file):
489     if not os.path.exists(file):
490         return None
491     else:
492         (type_base, type_mods) = split_p4_type(p4_type(file))
493         return p4_keywords_regexp_for_type(type_base, type_mods)
494
495 def setP4ExecBit(file, mode):
496     # Reopens an already open file and changes the execute bit to match
497     # the execute bit setting in the passed in mode.
498
499     p4Type = "+x"
500
501     if not isModeExec(mode):
502         p4Type = getP4OpenedType(file)
503         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
504         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
505         if p4Type[-1] == "+":
506             p4Type = p4Type[0:-1]
507
508     p4_reopen(p4Type, file)
509
510 def getP4OpenedType(file):
511     # Returns the perforce file type for the given file.
512
513     result = p4_read_pipe(["opened", wildcard_encode(file)])
514     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
515     if match:
516         return match.group(1)
517     else:
518         die("Could not determine file type for %s (result: '%s')" % (file, result))
519
520 # Return the set of all p4 labels
521 def getP4Labels(depotPaths):
522     labels = set()
523     if isinstance(depotPaths,basestring):
524         depotPaths = [depotPaths]
525
526     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
527         label = l['label']
528         labels.add(label)
529
530     return labels
531
532 # Return the set of all git tags
533 def getGitTags():
534     gitTags = set()
535     for line in read_pipe_lines(["git", "tag"]):
536         tag = line.strip()
537         gitTags.add(tag)
538     return gitTags
539
540 def diffTreePattern():
541     # This is a simple generator for the diff tree regex pattern. This could be
542     # a class variable if this and parseDiffTreeEntry were a part of a class.
543     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
544     while True:
545         yield pattern
546
547 def parseDiffTreeEntry(entry):
548     """Parses a single diff tree entry into its component elements.
549
550     See git-diff-tree(1) manpage for details about the format of the diff
551     output. This method returns a dictionary with the following elements:
552
553     src_mode - The mode of the source file
554     dst_mode - The mode of the destination file
555     src_sha1 - The sha1 for the source file
556     dst_sha1 - The sha1 fr the destination file
557     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
558     status_score - The score for the status (applicable for 'C' and 'R'
559                    statuses). This is None if there is no score.
560     src - The path for the source file.
561     dst - The path for the destination file. This is only present for
562           copy or renames. If it is not present, this is None.
563
564     If the pattern is not matched, None is returned."""
565
566     match = diffTreePattern().next().match(entry)
567     if match:
568         return {
569             'src_mode': match.group(1),
570             'dst_mode': match.group(2),
571             'src_sha1': match.group(3),
572             'dst_sha1': match.group(4),
573             'status': match.group(5),
574             'status_score': match.group(6),
575             'src': match.group(7),
576             'dst': match.group(10)
577         }
578     return None
579
580 def isModeExec(mode):
581     # Returns True if the given git mode represents an executable file,
582     # otherwise False.
583     return mode[-3:] == "755"
584
585 class P4Exception(Exception):
586     """ Base class for exceptions from the p4 client """
587     def __init__(self, exit_code):
588         self.p4ExitCode = exit_code
589
590 class P4ServerException(P4Exception):
591     """ Base class for exceptions where we get some kind of marshalled up result from the server """
592     def __init__(self, exit_code, p4_result):
593         super(P4ServerException, self).__init__(exit_code)
594         self.p4_result = p4_result
595         self.code = p4_result[0]['code']
596         self.data = p4_result[0]['data']
597
598 class P4RequestSizeException(P4ServerException):
599     """ One of the maxresults or maxscanrows errors """
600     def __init__(self, exit_code, p4_result, limit):
601         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
602         self.limit = limit
603
604 def isModeExecChanged(src_mode, dst_mode):
605     return isModeExec(src_mode) != isModeExec(dst_mode)
606
607 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
608         errors_as_exceptions=False):
609
610     if isinstance(cmd,basestring):
611         cmd = "-G " + cmd
612         expand = True
613     else:
614         cmd = ["-G"] + cmd
615         expand = False
616
617     cmd = p4_build_cmd(cmd)
618     if verbose:
619         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
620
621     # Use a temporary file to avoid deadlocks without
622     # subprocess.communicate(), which would put another copy
623     # of stdout into memory.
624     stdin_file = None
625     if stdin is not None:
626         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
627         if isinstance(stdin,basestring):
628             stdin_file.write(stdin)
629         else:
630             for i in stdin:
631                 stdin_file.write(i + '\n')
632         stdin_file.flush()
633         stdin_file.seek(0)
634
635     p4 = subprocess.Popen(cmd,
636                           shell=expand,
637                           stdin=stdin_file,
638                           stdout=subprocess.PIPE)
639
640     result = []
641     try:
642         while True:
643             entry = marshal.load(p4.stdout)
644             if skip_info:
645                 if 'code' in entry and entry['code'] == 'info':
646                     continue
647             if cb is not None:
648                 cb(entry)
649             else:
650                 result.append(entry)
651     except EOFError:
652         pass
653     exitCode = p4.wait()
654     if exitCode != 0:
655         if errors_as_exceptions:
656             if len(result) > 0:
657                 data = result[0].get('data')
658                 if data:
659                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
660                     if not m:
661                         m = re.search('Request too large \(over (\d+)\)', data)
662
663                     if m:
664                         limit = int(m.group(1))
665                         raise P4RequestSizeException(exitCode, result, limit)
666
667                 raise P4ServerException(exitCode, result)
668             else:
669                 raise P4Exception(exitCode)
670         else:
671             entry = {}
672             entry["p4ExitCode"] = exitCode
673             result.append(entry)
674
675     return result
676
677 def p4Cmd(cmd):
678     list = p4CmdList(cmd)
679     result = {}
680     for entry in list:
681         result.update(entry)
682     return result;
683
684 def p4Where(depotPath):
685     if not depotPath.endswith("/"):
686         depotPath += "/"
687     depotPathLong = depotPath + "..."
688     outputList = p4CmdList(["where", depotPathLong])
689     output = None
690     for entry in outputList:
691         if "depotFile" in entry:
692             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
693             # The base path always ends with "/...".
694             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
695                 output = entry
696                 break
697         elif "data" in entry:
698             data = entry.get("data")
699             space = data.find(" ")
700             if data[:space] == depotPath:
701                 output = entry
702                 break
703     if output == None:
704         return ""
705     if output["code"] == "error":
706         return ""
707     clientPath = ""
708     if "path" in output:
709         clientPath = output.get("path")
710     elif "data" in output:
711         data = output.get("data")
712         lastSpace = data.rfind(" ")
713         clientPath = data[lastSpace + 1:]
714
715     if clientPath.endswith("..."):
716         clientPath = clientPath[:-3]
717     return clientPath
718
719 def currentGitBranch():
720     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
721
722 def isValidGitDir(path):
723     return git_dir(path) != None
724
725 def parseRevision(ref):
726     return read_pipe("git rev-parse %s" % ref).strip()
727
728 def branchExists(ref):
729     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
730                      ignore_error=True)
731     return len(rev) > 0
732
733 def extractLogMessageFromGitCommit(commit):
734     logMessage = ""
735
736     ## fixme: title is first line of commit, not 1st paragraph.
737     foundTitle = False
738     for log in read_pipe_lines("git cat-file commit %s" % commit):
739        if not foundTitle:
740            if len(log) == 1:
741                foundTitle = True
742            continue
743
744        logMessage += log
745     return logMessage
746
747 def extractSettingsGitLog(log):
748     values = {}
749     for line in log.split("\n"):
750         line = line.strip()
751         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
752         if not m:
753             continue
754
755         assignments = m.group(1).split (':')
756         for a in assignments:
757             vals = a.split ('=')
758             key = vals[0].strip()
759             val = ('='.join (vals[1:])).strip()
760             if val.endswith ('\"') and val.startswith('"'):
761                 val = val[1:-1]
762
763             values[key] = val
764
765     paths = values.get("depot-paths")
766     if not paths:
767         paths = values.get("depot-path")
768     if paths:
769         values['depot-paths'] = paths.split(',')
770     return values
771
772 def gitBranchExists(branch):
773     proc = subprocess.Popen(["git", "rev-parse", branch],
774                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
775     return proc.wait() == 0;
776
777 def gitUpdateRef(ref, newvalue):
778     subprocess.check_call(["git", "update-ref", ref, newvalue])
779
780 def gitDeleteRef(ref):
781     subprocess.check_call(["git", "update-ref", "-d", ref])
782
783 _gitConfig = {}
784
785 def gitConfig(key, typeSpecifier=None):
786     if key not in _gitConfig:
787         cmd = [ "git", "config" ]
788         if typeSpecifier:
789             cmd += [ typeSpecifier ]
790         cmd += [ key ]
791         s = read_pipe(cmd, ignore_error=True)
792         _gitConfig[key] = s.strip()
793     return _gitConfig[key]
794
795 def gitConfigBool(key):
796     """Return a bool, using git config --bool.  It is True only if the
797        variable is set to true, and False if set to false or not present
798        in the config."""
799
800     if key not in _gitConfig:
801         _gitConfig[key] = gitConfig(key, '--bool') == "true"
802     return _gitConfig[key]
803
804 def gitConfigInt(key):
805     if key not in _gitConfig:
806         cmd = [ "git", "config", "--int", key ]
807         s = read_pipe(cmd, ignore_error=True)
808         v = s.strip()
809         try:
810             _gitConfig[key] = int(gitConfig(key, '--int'))
811         except ValueError:
812             _gitConfig[key] = None
813     return _gitConfig[key]
814
815 def gitConfigList(key):
816     if key not in _gitConfig:
817         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
818         _gitConfig[key] = s.strip().splitlines()
819         if _gitConfig[key] == ['']:
820             _gitConfig[key] = []
821     return _gitConfig[key]
822
823 def p4BranchesInGit(branchesAreInRemotes=True):
824     """Find all the branches whose names start with "p4/", looking
825        in remotes or heads as specified by the argument.  Return
826        a dictionary of { branch: revision } for each one found.
827        The branch names are the short names, without any
828        "p4/" prefix."""
829
830     branches = {}
831
832     cmdline = "git rev-parse --symbolic "
833     if branchesAreInRemotes:
834         cmdline += "--remotes"
835     else:
836         cmdline += "--branches"
837
838     for line in read_pipe_lines(cmdline):
839         line = line.strip()
840
841         # only import to p4/
842         if not line.startswith('p4/'):
843             continue
844         # special symbolic ref to p4/master
845         if line == "p4/HEAD":
846             continue
847
848         # strip off p4/ prefix
849         branch = line[len("p4/"):]
850
851         branches[branch] = parseRevision(line)
852
853     return branches
854
855 def branch_exists(branch):
856     """Make sure that the given ref name really exists."""
857
858     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
859     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
860     out, _ = p.communicate()
861     if p.returncode:
862         return False
863     # expect exactly one line of output: the branch name
864     return out.rstrip() == branch
865
866 def findUpstreamBranchPoint(head = "HEAD"):
867     branches = p4BranchesInGit()
868     # map from depot-path to branch name
869     branchByDepotPath = {}
870     for branch in branches.keys():
871         tip = branches[branch]
872         log = extractLogMessageFromGitCommit(tip)
873         settings = extractSettingsGitLog(log)
874         if "depot-paths" in settings:
875             paths = ",".join(settings["depot-paths"])
876             branchByDepotPath[paths] = "remotes/p4/" + branch
877
878     settings = None
879     parent = 0
880     while parent < 65535:
881         commit = head + "~%s" % parent
882         log = extractLogMessageFromGitCommit(commit)
883         settings = extractSettingsGitLog(log)
884         if "depot-paths" in settings:
885             paths = ",".join(settings["depot-paths"])
886             if paths in branchByDepotPath:
887                 return [branchByDepotPath[paths], settings]
888
889         parent = parent + 1
890
891     return ["", settings]
892
893 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
894     if not silent:
895         print("Creating/updating branch(es) in %s based on origin branch(es)"
896                % localRefPrefix)
897
898     originPrefix = "origin/p4/"
899
900     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
901         line = line.strip()
902         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
903             continue
904
905         headName = line[len(originPrefix):]
906         remoteHead = localRefPrefix + headName
907         originHead = line
908
909         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
910         if ('depot-paths' not in original
911             or 'change' not in original):
912             continue
913
914         update = False
915         if not gitBranchExists(remoteHead):
916             if verbose:
917                 print("creating %s" % remoteHead)
918             update = True
919         else:
920             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
921             if 'change' in settings:
922                 if settings['depot-paths'] == original['depot-paths']:
923                     originP4Change = int(original['change'])
924                     p4Change = int(settings['change'])
925                     if originP4Change > p4Change:
926                         print("%s (%s) is newer than %s (%s). "
927                                "Updating p4 branch from origin."
928                                % (originHead, originP4Change,
929                                   remoteHead, p4Change))
930                         update = True
931                 else:
932                     print("Ignoring: %s was imported from %s while "
933                            "%s was imported from %s"
934                            % (originHead, ','.join(original['depot-paths']),
935                               remoteHead, ','.join(settings['depot-paths'])))
936
937         if update:
938             system("git update-ref %s %s" % (remoteHead, originHead))
939
940 def originP4BranchesExist():
941         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
942
943
944 def p4ParseNumericChangeRange(parts):
945     changeStart = int(parts[0][1:])
946     if parts[1] == '#head':
947         changeEnd = p4_last_change()
948     else:
949         changeEnd = int(parts[1])
950
951     return (changeStart, changeEnd)
952
953 def chooseBlockSize(blockSize):
954     if blockSize:
955         return blockSize
956     else:
957         return defaultBlockSize
958
959 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
960     assert depotPaths
961
962     # Parse the change range into start and end. Try to find integer
963     # revision ranges as these can be broken up into blocks to avoid
964     # hitting server-side limits (maxrows, maxscanresults). But if
965     # that doesn't work, fall back to using the raw revision specifier
966     # strings, without using block mode.
967
968     if changeRange is None or changeRange == '':
969         changeStart = 1
970         changeEnd = p4_last_change()
971         block_size = chooseBlockSize(requestedBlockSize)
972     else:
973         parts = changeRange.split(',')
974         assert len(parts) == 2
975         try:
976             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
977             block_size = chooseBlockSize(requestedBlockSize)
978         except ValueError:
979             changeStart = parts[0][1:]
980             changeEnd = parts[1]
981             if requestedBlockSize:
982                 die("cannot use --changes-block-size with non-numeric revisions")
983             block_size = None
984
985     changes = set()
986
987     # Retrieve changes a block at a time, to prevent running
988     # into a MaxResults/MaxScanRows error from the server. If
989     # we _do_ hit one of those errors, turn down the block size
990
991     while True:
992         cmd = ['changes']
993
994         if block_size:
995             end = min(changeEnd, changeStart + block_size)
996             revisionRange = "%d,%d" % (changeStart, end)
997         else:
998             revisionRange = "%s,%s" % (changeStart, changeEnd)
999
1000         for p in depotPaths:
1001             cmd += ["%s...@%s" % (p, revisionRange)]
1002
1003         # fetch the changes
1004         try:
1005             result = p4CmdList(cmd, errors_as_exceptions=True)
1006         except P4RequestSizeException as e:
1007             if not block_size:
1008                 block_size = e.limit
1009             elif block_size > e.limit:
1010                 block_size = e.limit
1011             else:
1012                 block_size = max(2, block_size // 2)
1013
1014             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1015             continue
1016         except P4Exception as e:
1017             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1018
1019         # Insert changes in chronological order
1020         for entry in reversed(result):
1021             if 'change' not in entry:
1022                 continue
1023             changes.add(int(entry['change']))
1024
1025         if not block_size:
1026             break
1027
1028         if end >= changeEnd:
1029             break
1030
1031         changeStart = end + 1
1032
1033     changes = sorted(changes)
1034     return changes
1035
1036 def p4PathStartsWith(path, prefix):
1037     # This method tries to remedy a potential mixed-case issue:
1038     #
1039     # If UserA adds  //depot/DirA/file1
1040     # and UserB adds //depot/dira/file2
1041     #
1042     # we may or may not have a problem. If you have core.ignorecase=true,
1043     # we treat DirA and dira as the same directory
1044     if gitConfigBool("core.ignorecase"):
1045         return path.lower().startswith(prefix.lower())
1046     return path.startswith(prefix)
1047
1048 def getClientSpec():
1049     """Look at the p4 client spec, create a View() object that contains
1050        all the mappings, and return it."""
1051
1052     specList = p4CmdList("client -o")
1053     if len(specList) != 1:
1054         die('Output from "client -o" is %d lines, expecting 1' %
1055             len(specList))
1056
1057     # dictionary of all client parameters
1058     entry = specList[0]
1059
1060     # the //client/ name
1061     client_name = entry["Client"]
1062
1063     # just the keys that start with "View"
1064     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1065
1066     # hold this new View
1067     view = View(client_name)
1068
1069     # append the lines, in order, to the view
1070     for view_num in range(len(view_keys)):
1071         k = "View%d" % view_num
1072         if k not in view_keys:
1073             die("Expected view key %s missing" % k)
1074         view.append(entry[k])
1075
1076     return view
1077
1078 def getClientRoot():
1079     """Grab the client directory."""
1080
1081     output = p4CmdList("client -o")
1082     if len(output) != 1:
1083         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1084
1085     entry = output[0]
1086     if "Root" not in entry:
1087         die('Client has no "Root"')
1088
1089     return entry["Root"]
1090
1091 #
1092 # P4 wildcards are not allowed in filenames.  P4 complains
1093 # if you simply add them, but you can force it with "-f", in
1094 # which case it translates them into %xx encoding internally.
1095 #
1096 def wildcard_decode(path):
1097     # Search for and fix just these four characters.  Do % last so
1098     # that fixing it does not inadvertently create new %-escapes.
1099     # Cannot have * in a filename in windows; untested as to
1100     # what p4 would do in such a case.
1101     if not platform.system() == "Windows":
1102         path = path.replace("%2A", "*")
1103     path = path.replace("%23", "#") \
1104                .replace("%40", "@") \
1105                .replace("%25", "%")
1106     return path
1107
1108 def wildcard_encode(path):
1109     # do % first to avoid double-encoding the %s introduced here
1110     path = path.replace("%", "%25") \
1111                .replace("*", "%2A") \
1112                .replace("#", "%23") \
1113                .replace("@", "%40")
1114     return path
1115
1116 def wildcard_present(path):
1117     m = re.search("[*#@%]", path)
1118     return m is not None
1119
1120 class LargeFileSystem(object):
1121     """Base class for large file system support."""
1122
1123     def __init__(self, writeToGitStream):
1124         self.largeFiles = set()
1125         self.writeToGitStream = writeToGitStream
1126
1127     def generatePointer(self, cloneDestination, contentFile):
1128         """Return the content of a pointer file that is stored in Git instead of
1129            the actual content."""
1130         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1131
1132     def pushFile(self, localLargeFile):
1133         """Push the actual content which is not stored in the Git repository to
1134            a server."""
1135         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1136
1137     def hasLargeFileExtension(self, relPath):
1138         return reduce(
1139             lambda a, b: a or b,
1140             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1141             False
1142         )
1143
1144     def generateTempFile(self, contents):
1145         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146         for d in contents:
1147             contentFile.write(d)
1148         contentFile.close()
1149         return contentFile.name
1150
1151     def exceedsLargeFileThreshold(self, relPath, contents):
1152         if gitConfigInt('git-p4.largeFileThreshold'):
1153             contentsSize = sum(len(d) for d in contents)
1154             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1155                 return True
1156         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1157             contentsSize = sum(len(d) for d in contents)
1158             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159                 return False
1160             contentTempFile = self.generateTempFile(contents)
1161             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1162             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1163             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1164             zf.close()
1165             compressedContentsSize = zf.infolist()[0].compress_size
1166             os.remove(contentTempFile)
1167             os.remove(compressedContentFile.name)
1168             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169                 return True
1170         return False
1171
1172     def addLargeFile(self, relPath):
1173         self.largeFiles.add(relPath)
1174
1175     def removeLargeFile(self, relPath):
1176         self.largeFiles.remove(relPath)
1177
1178     def isLargeFile(self, relPath):
1179         return relPath in self.largeFiles
1180
1181     def processContent(self, git_mode, relPath, contents):
1182         """Processes the content of git fast import. This method decides if a
1183            file is stored in the large file system and handles all necessary
1184            steps."""
1185         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186             contentTempFile = self.generateTempFile(contents)
1187             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188             if pointer_git_mode:
1189                 git_mode = pointer_git_mode
1190             if localLargeFile:
1191                 # Move temp file to final location in large file system
1192                 largeFileDir = os.path.dirname(localLargeFile)
1193                 if not os.path.isdir(largeFileDir):
1194                     os.makedirs(largeFileDir)
1195                 shutil.move(contentTempFile, localLargeFile)
1196                 self.addLargeFile(relPath)
1197                 if gitConfigBool('git-p4.largeFilePush'):
1198                     self.pushFile(localLargeFile)
1199                 if verbose:
1200                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201         return (git_mode, contents)
1202
1203 class MockLFS(LargeFileSystem):
1204     """Mock large file system for testing."""
1205
1206     def generatePointer(self, contentFile):
1207         """The pointer content is the original content prefixed with "pointer-".
1208            The local filename of the large file storage is derived from the file content.
1209            """
1210         with open(contentFile, 'r') as f:
1211             content = next(f)
1212             gitMode = '100644'
1213             pointerContents = 'pointer-' + content
1214             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215             return (gitMode, pointerContents, localLargeFile)
1216
1217     def pushFile(self, localLargeFile):
1218         """The remote filename of the large file storage is the same as the local
1219            one but in a different directory.
1220            """
1221         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222         if not os.path.exists(remotePath):
1223             os.makedirs(remotePath)
1224         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1225
1226 class GitLFS(LargeFileSystem):
1227     """Git LFS as backend for the git-p4 large file system.
1228        See https://git-lfs.github.com/ for details."""
1229
1230     def __init__(self, *args):
1231         LargeFileSystem.__init__(self, *args)
1232         self.baseGitAttributes = []
1233
1234     def generatePointer(self, contentFile):
1235         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236            mode and content which is stored in the Git repository instead of
1237            the actual content. Return also the new location of the actual
1238            content.
1239            """
1240         if os.path.getsize(contentFile) == 0:
1241             return (None, '', None)
1242
1243         pointerProcess = subprocess.Popen(
1244             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245             stdout=subprocess.PIPE
1246         )
1247         pointerFile = pointerProcess.stdout.read()
1248         if pointerProcess.wait():
1249             os.remove(contentFile)
1250             die('git-lfs pointer command failed. Did you install the extension?')
1251
1252         # Git LFS removed the preamble in the output of the 'pointer' command
1253         # starting from version 1.2.0. Check for the preamble here to support
1254         # earlier versions.
1255         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256         if pointerFile.startswith('Git LFS pointer for'):
1257             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1258
1259         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260         localLargeFile = os.path.join(
1261             os.getcwd(),
1262             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1263             oid,
1264         )
1265         # LFS Spec states that pointer files should not have the executable bit set.
1266         gitMode = '100644'
1267         return (gitMode, pointerFile, localLargeFile)
1268
1269     def pushFile(self, localLargeFile):
1270         uploadProcess = subprocess.Popen(
1271             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1272         )
1273         if uploadProcess.wait():
1274             die('git-lfs push command failed. Did you define a remote?')
1275
1276     def generateGitAttributes(self):
1277         return (
1278             self.baseGitAttributes +
1279             [
1280                 '\n',
1281                 '#\n',
1282                 '# Git LFS (see https://git-lfs.github.com/)\n',
1283                 '#\n',
1284             ] +
1285             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1286                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1287             ] +
1288             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1289                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1290             ]
1291         )
1292
1293     def addLargeFile(self, relPath):
1294         LargeFileSystem.addLargeFile(self, relPath)
1295         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1296
1297     def removeLargeFile(self, relPath):
1298         LargeFileSystem.removeLargeFile(self, relPath)
1299         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1300
1301     def processContent(self, git_mode, relPath, contents):
1302         if relPath == '.gitattributes':
1303             self.baseGitAttributes = contents
1304             return (git_mode, self.generateGitAttributes())
1305         else:
1306             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1307
1308 class Command:
1309     def __init__(self):
1310         self.usage = "usage: %prog [options]"
1311         self.needsGit = True
1312         self.verbose = False
1313
1314     # This is required for the "append" cloneExclude action
1315     def ensure_value(self, attr, value):
1316         if not hasattr(self, attr) or getattr(self, attr) is None:
1317             setattr(self, attr, value)
1318         return getattr(self, attr)
1319
1320 class P4UserMap:
1321     def __init__(self):
1322         self.userMapFromPerforceServer = False
1323         self.myP4UserId = None
1324
1325     def p4UserId(self):
1326         if self.myP4UserId:
1327             return self.myP4UserId
1328
1329         results = p4CmdList("user -o")
1330         for r in results:
1331             if 'User' in r:
1332                 self.myP4UserId = r['User']
1333                 return r['User']
1334         die("Could not find your p4 user id")
1335
1336     def p4UserIsMe(self, p4User):
1337         # return True if the given p4 user is actually me
1338         me = self.p4UserId()
1339         if not p4User or p4User != me:
1340             return False
1341         else:
1342             return True
1343
1344     def getUserCacheFilename(self):
1345         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1346         return home + "/.gitp4-usercache.txt"
1347
1348     def getUserMapFromPerforceServer(self):
1349         if self.userMapFromPerforceServer:
1350             return
1351         self.users = {}
1352         self.emails = {}
1353
1354         for output in p4CmdList("users"):
1355             if "User" not in output:
1356                 continue
1357             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1358             self.emails[output["Email"]] = output["User"]
1359
1360         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1361         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1362             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1363             if mapUser and len(mapUser[0]) == 3:
1364                 user = mapUser[0][0]
1365                 fullname = mapUser[0][1]
1366                 email = mapUser[0][2]
1367                 self.users[user] = fullname + " <" + email + ">"
1368                 self.emails[email] = user
1369
1370         s = ''
1371         for (key, val) in self.users.items():
1372             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1373
1374         open(self.getUserCacheFilename(), "wb").write(s)
1375         self.userMapFromPerforceServer = True
1376
1377     def loadUserMapFromCache(self):
1378         self.users = {}
1379         self.userMapFromPerforceServer = False
1380         try:
1381             cache = open(self.getUserCacheFilename(), "rb")
1382             lines = cache.readlines()
1383             cache.close()
1384             for line in lines:
1385                 entry = line.strip().split("\t")
1386                 self.users[entry[0]] = entry[1]
1387         except IOError:
1388             self.getUserMapFromPerforceServer()
1389
1390 class P4Debug(Command):
1391     def __init__(self):
1392         Command.__init__(self)
1393         self.options = []
1394         self.description = "A tool to debug the output of p4 -G."
1395         self.needsGit = False
1396
1397     def run(self, args):
1398         j = 0
1399         for output in p4CmdList(args):
1400             print('Element: %d' % j)
1401             j += 1
1402             print(output)
1403         return True
1404
1405 class P4RollBack(Command):
1406     def __init__(self):
1407         Command.__init__(self)
1408         self.options = [
1409             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1410         ]
1411         self.description = "A tool to debug the multi-branch import. Don't use :)"
1412         self.rollbackLocalBranches = False
1413
1414     def run(self, args):
1415         if len(args) != 1:
1416             return False
1417         maxChange = int(args[0])
1418
1419         if "p4ExitCode" in p4Cmd("changes -m 1"):
1420             die("Problems executing p4");
1421
1422         if self.rollbackLocalBranches:
1423             refPrefix = "refs/heads/"
1424             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1425         else:
1426             refPrefix = "refs/remotes/"
1427             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1428
1429         for line in lines:
1430             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1431                 line = line.strip()
1432                 ref = refPrefix + line
1433                 log = extractLogMessageFromGitCommit(ref)
1434                 settings = extractSettingsGitLog(log)
1435
1436                 depotPaths = settings['depot-paths']
1437                 change = settings['change']
1438
1439                 changed = False
1440
1441                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1442                                                            for p in depotPaths]))) == 0:
1443                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1444                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1445                     continue
1446
1447                 while change and int(change) > maxChange:
1448                     changed = True
1449                     if self.verbose:
1450                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1451                     system("git update-ref %s \"%s^\"" % (ref, ref))
1452                     log = extractLogMessageFromGitCommit(ref)
1453                     settings =  extractSettingsGitLog(log)
1454
1455
1456                     depotPaths = settings['depot-paths']
1457                     change = settings['change']
1458
1459                 if changed:
1460                     print("%s rewound to %s" % (ref, change))
1461
1462         return True
1463
1464 class P4Submit(Command, P4UserMap):
1465
1466     conflict_behavior_choices = ("ask", "skip", "quit")
1467
1468     def __init__(self):
1469         Command.__init__(self)
1470         P4UserMap.__init__(self)
1471         self.options = [
1472                 optparse.make_option("--origin", dest="origin"),
1473                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1474                 # preserve the user, requires relevant p4 permissions
1475                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1476                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1477                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1478                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1479                 optparse.make_option("--conflict", dest="conflict_behavior",
1480                                      choices=self.conflict_behavior_choices),
1481                 optparse.make_option("--branch", dest="branch"),
1482                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1483                                      help="Shelve instead of submit. Shelved files are reverted, "
1484                                      "restoring the workspace to the state before the shelve"),
1485                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1486                                      metavar="CHANGELIST",
1487                                      help="update an existing shelved changelist, implies --shelve, "
1488                                            "repeat in-order for multiple shelved changelists"),
1489                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1490                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1491                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1492                                      help="Disable rebase after submit is completed. Can be useful if you "
1493                                      "work from a local git branch that is not master"),
1494                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1495                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1496         ]
1497         self.description = "Submit changes from git to the perforce depot."
1498         self.usage += " [name of git branch to submit into perforce depot]"
1499         self.origin = ""
1500         self.detectRenames = False
1501         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1502         self.dry_run = False
1503         self.shelve = False
1504         self.update_shelve = list()
1505         self.commit = ""
1506         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1507         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1508         self.prepare_p4_only = False
1509         self.conflict_behavior = None
1510         self.isWindows = (platform.system() == "Windows")
1511         self.exportLabels = False
1512         self.p4HasMoveCommand = p4_has_move_command()
1513         self.branch = None
1514
1515         if gitConfig('git-p4.largeFileSystem'):
1516             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1517
1518     def check(self):
1519         if len(p4CmdList("opened ...")) > 0:
1520             die("You have files opened with perforce! Close them before starting the sync.")
1521
1522     def separate_jobs_from_description(self, message):
1523         """Extract and return a possible Jobs field in the commit
1524            message.  It goes into a separate section in the p4 change
1525            specification.
1526
1527            A jobs line starts with "Jobs:" and looks like a new field
1528            in a form.  Values are white-space separated on the same
1529            line or on following lines that start with a tab.
1530
1531            This does not parse and extract the full git commit message
1532            like a p4 form.  It just sees the Jobs: line as a marker
1533            to pass everything from then on directly into the p4 form,
1534            but outside the description section.
1535
1536            Return a tuple (stripped log message, jobs string)."""
1537
1538         m = re.search(r'^Jobs:', message, re.MULTILINE)
1539         if m is None:
1540             return (message, None)
1541
1542         jobtext = message[m.start():]
1543         stripped_message = message[:m.start()].rstrip()
1544         return (stripped_message, jobtext)
1545
1546     def prepareLogMessage(self, template, message, jobs):
1547         """Edits the template returned from "p4 change -o" to insert
1548            the message in the Description field, and the jobs text in
1549            the Jobs field."""
1550         result = ""
1551
1552         inDescriptionSection = False
1553
1554         for line in template.split("\n"):
1555             if line.startswith("#"):
1556                 result += line + "\n"
1557                 continue
1558
1559             if inDescriptionSection:
1560                 if line.startswith("Files:") or line.startswith("Jobs:"):
1561                     inDescriptionSection = False
1562                     # insert Jobs section
1563                     if jobs:
1564                         result += jobs + "\n"
1565                 else:
1566                     continue
1567             else:
1568                 if line.startswith("Description:"):
1569                     inDescriptionSection = True
1570                     line += "\n"
1571                     for messageLine in message.split("\n"):
1572                         line += "\t" + messageLine + "\n"
1573
1574             result += line + "\n"
1575
1576         return result
1577
1578     def patchRCSKeywords(self, file, pattern):
1579         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1580         (handle, outFileName) = tempfile.mkstemp(dir='.')
1581         try:
1582             outFile = os.fdopen(handle, "w+")
1583             inFile = open(file, "r")
1584             regexp = re.compile(pattern, re.VERBOSE)
1585             for line in inFile.readlines():
1586                 line = regexp.sub(r'$\1$', line)
1587                 outFile.write(line)
1588             inFile.close()
1589             outFile.close()
1590             # Forcibly overwrite the original file
1591             os.unlink(file)
1592             shutil.move(outFileName, file)
1593         except:
1594             # cleanup our temporary file
1595             os.unlink(outFileName)
1596             print("Failed to strip RCS keywords in %s" % file)
1597             raise
1598
1599         print("Patched up RCS keywords in %s" % file)
1600
1601     def p4UserForCommit(self,id):
1602         # Return the tuple (perforce user,git email) for a given git commit id
1603         self.getUserMapFromPerforceServer()
1604         gitEmail = read_pipe(["git", "log", "--max-count=1",
1605                               "--format=%ae", id])
1606         gitEmail = gitEmail.strip()
1607         if gitEmail not in self.emails:
1608             return (None,gitEmail)
1609         else:
1610             return (self.emails[gitEmail],gitEmail)
1611
1612     def checkValidP4Users(self,commits):
1613         # check if any git authors cannot be mapped to p4 users
1614         for id in commits:
1615             (user,email) = self.p4UserForCommit(id)
1616             if not user:
1617                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1618                 if gitConfigBool("git-p4.allowMissingP4Users"):
1619                     print("%s" % msg)
1620                 else:
1621                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1622
1623     def lastP4Changelist(self):
1624         # Get back the last changelist number submitted in this client spec. This
1625         # then gets used to patch up the username in the change. If the same
1626         # client spec is being used by multiple processes then this might go
1627         # wrong.
1628         results = p4CmdList("client -o")        # find the current client
1629         client = None
1630         for r in results:
1631             if 'Client' in r:
1632                 client = r['Client']
1633                 break
1634         if not client:
1635             die("could not get client spec")
1636         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1637         for r in results:
1638             if 'change' in r:
1639                 return r['change']
1640         die("Could not get changelist number for last submit - cannot patch up user details")
1641
1642     def modifyChangelistUser(self, changelist, newUser):
1643         # fixup the user field of a changelist after it has been submitted.
1644         changes = p4CmdList("change -o %s" % changelist)
1645         if len(changes) != 1:
1646             die("Bad output from p4 change modifying %s to user %s" %
1647                 (changelist, newUser))
1648
1649         c = changes[0]
1650         if c['User'] == newUser: return   # nothing to do
1651         c['User'] = newUser
1652         input = marshal.dumps(c)
1653
1654         result = p4CmdList("change -f -i", stdin=input)
1655         for r in result:
1656             if 'code' in r:
1657                 if r['code'] == 'error':
1658                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1659             if 'data' in r:
1660                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1661                 return
1662         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1663
1664     def canChangeChangelists(self):
1665         # check to see if we have p4 admin or super-user permissions, either of
1666         # which are required to modify changelists.
1667         results = p4CmdList(["protects", self.depotPath])
1668         for r in results:
1669             if 'perm' in r:
1670                 if r['perm'] == 'admin':
1671                     return 1
1672                 if r['perm'] == 'super':
1673                     return 1
1674         return 0
1675
1676     def prepareSubmitTemplate(self, changelist=None):
1677         """Run "p4 change -o" to grab a change specification template.
1678            This does not use "p4 -G", as it is nice to keep the submission
1679            template in original order, since a human might edit it.
1680
1681            Remove lines in the Files section that show changes to files
1682            outside the depot path we're committing into."""
1683
1684         [upstream, settings] = findUpstreamBranchPoint()
1685
1686         template = """\
1687 # A Perforce Change Specification.
1688 #
1689 #  Change:      The change number. 'new' on a new changelist.
1690 #  Date:        The date this specification was last modified.
1691 #  Client:      The client on which the changelist was created.  Read-only.
1692 #  User:        The user who created the changelist.
1693 #  Status:      Either 'pending' or 'submitted'. Read-only.
1694 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1695 #  Description: Comments about the changelist.  Required.
1696 #  Jobs:        What opened jobs are to be closed by this changelist.
1697 #               You may delete jobs from this list.  (New changelists only.)
1698 #  Files:       What opened files from the default changelist are to be added
1699 #               to this changelist.  You may delete files from this list.
1700 #               (New changelists only.)
1701 """
1702         files_list = []
1703         inFilesSection = False
1704         change_entry = None
1705         args = ['change', '-o']
1706         if changelist:
1707             args.append(str(changelist))
1708         for entry in p4CmdList(args):
1709             if 'code' not in entry:
1710                 continue
1711             if entry['code'] == 'stat':
1712                 change_entry = entry
1713                 break
1714         if not change_entry:
1715             die('Failed to decode output of p4 change -o')
1716         for key, value in change_entry.iteritems():
1717             if key.startswith('File'):
1718                 if 'depot-paths' in settings:
1719                     if not [p for p in settings['depot-paths']
1720                             if p4PathStartsWith(value, p)]:
1721                         continue
1722                 else:
1723                     if not p4PathStartsWith(value, self.depotPath):
1724                         continue
1725                 files_list.append(value)
1726                 continue
1727         # Output in the order expected by prepareLogMessage
1728         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1729             if key not in change_entry:
1730                 continue
1731             template += '\n'
1732             template += key + ':'
1733             if key == 'Description':
1734                 template += '\n'
1735             for field_line in change_entry[key].splitlines():
1736                 template += '\t'+field_line+'\n'
1737         if len(files_list) > 0:
1738             template += '\n'
1739             template += 'Files:\n'
1740         for path in files_list:
1741             template += '\t'+path+'\n'
1742         return template
1743
1744     def edit_template(self, template_file):
1745         """Invoke the editor to let the user change the submission
1746            message.  Return true if okay to continue with the submit."""
1747
1748         # if configured to skip the editing part, just submit
1749         if gitConfigBool("git-p4.skipSubmitEdit"):
1750             return True
1751
1752         # look at the modification time, to check later if the user saved
1753         # the file
1754         mtime = os.stat(template_file).st_mtime
1755
1756         # invoke the editor
1757         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1758             editor = os.environ.get("P4EDITOR")
1759         else:
1760             editor = read_pipe("git var GIT_EDITOR").strip()
1761         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1762
1763         # If the file was not saved, prompt to see if this patch should
1764         # be skipped.  But skip this verification step if configured so.
1765         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1766             return True
1767
1768         # modification time updated means user saved the file
1769         if os.stat(template_file).st_mtime > mtime:
1770             return True
1771
1772         while True:
1773             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1774             if response == 'y':
1775                 return True
1776             if response == 'n':
1777                 return False
1778
1779     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1780         # diff
1781         if "P4DIFF" in os.environ:
1782             del(os.environ["P4DIFF"])
1783         diff = ""
1784         for editedFile in editedFiles:
1785             diff += p4_read_pipe(['diff', '-du',
1786                                   wildcard_encode(editedFile)])
1787
1788         # new file diff
1789         newdiff = ""
1790         for newFile in filesToAdd:
1791             newdiff += "==== new file ====\n"
1792             newdiff += "--- /dev/null\n"
1793             newdiff += "+++ %s\n" % newFile
1794
1795             is_link = os.path.islink(newFile)
1796             expect_link = newFile in symlinks
1797
1798             if is_link and expect_link:
1799                 newdiff += "+%s\n" % os.readlink(newFile)
1800             else:
1801                 f = open(newFile, "r")
1802                 for line in f.readlines():
1803                     newdiff += "+" + line
1804                 f.close()
1805
1806         return (diff + newdiff).replace('\r\n', '\n')
1807
1808     def applyCommit(self, id):
1809         """Apply one commit, return True if it succeeded."""
1810
1811         print("Applying", read_pipe(["git", "show", "-s",
1812                                      "--format=format:%h %s", id]))
1813
1814         (p4User, gitEmail) = self.p4UserForCommit(id)
1815
1816         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1817         filesToAdd = set()
1818         filesToChangeType = set()
1819         filesToDelete = set()
1820         editedFiles = set()
1821         pureRenameCopy = set()
1822         symlinks = set()
1823         filesToChangeExecBit = {}
1824         all_files = list()
1825
1826         for line in diff:
1827             diff = parseDiffTreeEntry(line)
1828             modifier = diff['status']
1829             path = diff['src']
1830             all_files.append(path)
1831
1832             if modifier == "M":
1833                 p4_edit(path)
1834                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1835                     filesToChangeExecBit[path] = diff['dst_mode']
1836                 editedFiles.add(path)
1837             elif modifier == "A":
1838                 filesToAdd.add(path)
1839                 filesToChangeExecBit[path] = diff['dst_mode']
1840                 if path in filesToDelete:
1841                     filesToDelete.remove(path)
1842
1843                 dst_mode = int(diff['dst_mode'], 8)
1844                 if dst_mode == 0o120000:
1845                     symlinks.add(path)
1846
1847             elif modifier == "D":
1848                 filesToDelete.add(path)
1849                 if path in filesToAdd:
1850                     filesToAdd.remove(path)
1851             elif modifier == "C":
1852                 src, dest = diff['src'], diff['dst']
1853                 p4_integrate(src, dest)
1854                 pureRenameCopy.add(dest)
1855                 if diff['src_sha1'] != diff['dst_sha1']:
1856                     p4_edit(dest)
1857                     pureRenameCopy.discard(dest)
1858                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1859                     p4_edit(dest)
1860                     pureRenameCopy.discard(dest)
1861                     filesToChangeExecBit[dest] = diff['dst_mode']
1862                 if self.isWindows:
1863                     # turn off read-only attribute
1864                     os.chmod(dest, stat.S_IWRITE)
1865                 os.unlink(dest)
1866                 editedFiles.add(dest)
1867             elif modifier == "R":
1868                 src, dest = diff['src'], diff['dst']
1869                 if self.p4HasMoveCommand:
1870                     p4_edit(src)        # src must be open before move
1871                     p4_move(src, dest)  # opens for (move/delete, move/add)
1872                 else:
1873                     p4_integrate(src, dest)
1874                     if diff['src_sha1'] != diff['dst_sha1']:
1875                         p4_edit(dest)
1876                     else:
1877                         pureRenameCopy.add(dest)
1878                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1879                     if not self.p4HasMoveCommand:
1880                         p4_edit(dest)   # with move: already open, writable
1881                     filesToChangeExecBit[dest] = diff['dst_mode']
1882                 if not self.p4HasMoveCommand:
1883                     if self.isWindows:
1884                         os.chmod(dest, stat.S_IWRITE)
1885                     os.unlink(dest)
1886                     filesToDelete.add(src)
1887                 editedFiles.add(dest)
1888             elif modifier == "T":
1889                 filesToChangeType.add(path)
1890             else:
1891                 die("unknown modifier %s for %s" % (modifier, path))
1892
1893         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1894         patchcmd = diffcmd + " | git apply "
1895         tryPatchCmd = patchcmd + "--check -"
1896         applyPatchCmd = patchcmd + "--check --apply -"
1897         patch_succeeded = True
1898
1899         if os.system(tryPatchCmd) != 0:
1900             fixed_rcs_keywords = False
1901             patch_succeeded = False
1902             print("Unfortunately applying the change failed!")
1903
1904             # Patch failed, maybe it's just RCS keyword woes. Look through
1905             # the patch to see if that's possible.
1906             if gitConfigBool("git-p4.attemptRCSCleanup"):
1907                 file = None
1908                 pattern = None
1909                 kwfiles = {}
1910                 for file in editedFiles | filesToDelete:
1911                     # did this file's delta contain RCS keywords?
1912                     pattern = p4_keywords_regexp_for_file(file)
1913
1914                     if pattern:
1915                         # this file is a possibility...look for RCS keywords.
1916                         regexp = re.compile(pattern, re.VERBOSE)
1917                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1918                             if regexp.search(line):
1919                                 if verbose:
1920                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1921                                 kwfiles[file] = pattern
1922                                 break
1923
1924                 for file in kwfiles:
1925                     if verbose:
1926                         print("zapping %s with %s" % (line,pattern))
1927                     # File is being deleted, so not open in p4.  Must
1928                     # disable the read-only bit on windows.
1929                     if self.isWindows and file not in editedFiles:
1930                         os.chmod(file, stat.S_IWRITE)
1931                     self.patchRCSKeywords(file, kwfiles[file])
1932                     fixed_rcs_keywords = True
1933
1934             if fixed_rcs_keywords:
1935                 print("Retrying the patch with RCS keywords cleaned up")
1936                 if os.system(tryPatchCmd) == 0:
1937                     patch_succeeded = True
1938
1939         if not patch_succeeded:
1940             for f in editedFiles:
1941                 p4_revert(f)
1942             return False
1943
1944         #
1945         # Apply the patch for real, and do add/delete/+x handling.
1946         #
1947         system(applyPatchCmd)
1948
1949         for f in filesToChangeType:
1950             p4_edit(f, "-t", "auto")
1951         for f in filesToAdd:
1952             p4_add(f)
1953         for f in filesToDelete:
1954             p4_revert(f)
1955             p4_delete(f)
1956
1957         # Set/clear executable bits
1958         for f in filesToChangeExecBit.keys():
1959             mode = filesToChangeExecBit[f]
1960             setP4ExecBit(f, mode)
1961
1962         update_shelve = 0
1963         if len(self.update_shelve) > 0:
1964             update_shelve = self.update_shelve.pop(0)
1965             p4_reopen_in_change(update_shelve, all_files)
1966
1967         #
1968         # Build p4 change description, starting with the contents
1969         # of the git commit message.
1970         #
1971         logMessage = extractLogMessageFromGitCommit(id)
1972         logMessage = logMessage.strip()
1973         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1974
1975         template = self.prepareSubmitTemplate(update_shelve)
1976         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1977
1978         if self.preserveUser:
1979            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1980
1981         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1982             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1983             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1984             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1985
1986         separatorLine = "######## everything below this line is just the diff #######\n"
1987         if not self.prepare_p4_only:
1988             submitTemplate += separatorLine
1989             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1990
1991         (handle, fileName) = tempfile.mkstemp()
1992         tmpFile = os.fdopen(handle, "w+b")
1993         if self.isWindows:
1994             submitTemplate = submitTemplate.replace("\n", "\r\n")
1995         tmpFile.write(submitTemplate)
1996         tmpFile.close()
1997
1998         if self.prepare_p4_only:
1999             #
2000             # Leave the p4 tree prepared, and the submit template around
2001             # and let the user decide what to do next
2002             #
2003             print()
2004             print("P4 workspace prepared for submission.")
2005             print("To submit or revert, go to client workspace")
2006             print("  " + self.clientPath)
2007             print()
2008             print("To submit, use \"p4 submit\" to write a new description,")
2009             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2010                   " \"git p4\"." % fileName)
2011             print("You can delete the file \"%s\" when finished." % fileName)
2012
2013             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2014                 print("To preserve change ownership by user %s, you must\n" \
2015                       "do \"p4 change -f <change>\" after submitting and\n" \
2016                       "edit the User field.")
2017             if pureRenameCopy:
2018                 print("After submitting, renamed files must be re-synced.")
2019                 print("Invoke \"p4 sync -f\" on each of these files:")
2020                 for f in pureRenameCopy:
2021                     print("  " + f)
2022
2023             print()
2024             print("To revert the changes, use \"p4 revert ...\", and delete")
2025             print("the submit template file \"%s\"" % fileName)
2026             if filesToAdd:
2027                 print("Since the commit adds new files, they must be deleted:")
2028                 for f in filesToAdd:
2029                     print("  " + f)
2030             print()
2031             return True
2032
2033         #
2034         # Let the user edit the change description, then submit it.
2035         #
2036         submitted = False
2037
2038         try:
2039             if self.edit_template(fileName):
2040                 # read the edited message and submit
2041                 tmpFile = open(fileName, "rb")
2042                 message = tmpFile.read()
2043                 tmpFile.close()
2044                 if self.isWindows:
2045                     message = message.replace("\r\n", "\n")
2046                 submitTemplate = message[:message.index(separatorLine)]
2047
2048                 if update_shelve:
2049                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2050                 elif self.shelve:
2051                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2052                 else:
2053                     p4_write_pipe(['submit', '-i'], submitTemplate)
2054                     # The rename/copy happened by applying a patch that created a
2055                     # new file.  This leaves it writable, which confuses p4.
2056                     for f in pureRenameCopy:
2057                         p4_sync(f, "-f")
2058
2059                 if self.preserveUser:
2060                     if p4User:
2061                         # Get last changelist number. Cannot easily get it from
2062                         # the submit command output as the output is
2063                         # unmarshalled.
2064                         changelist = self.lastP4Changelist()
2065                         self.modifyChangelistUser(changelist, p4User)
2066
2067                 submitted = True
2068
2069         finally:
2070             # skip this patch
2071             if not submitted or self.shelve:
2072                 if self.shelve:
2073                     print ("Reverting shelved files.")
2074                 else:
2075                     print ("Submission cancelled, undoing p4 changes.")
2076                 for f in editedFiles | filesToDelete:
2077                     p4_revert(f)
2078                 for f in filesToAdd:
2079                     p4_revert(f)
2080                     os.remove(f)
2081
2082         os.remove(fileName)
2083         return submitted
2084
2085     # Export git tags as p4 labels. Create a p4 label and then tag
2086     # with that.
2087     def exportGitTags(self, gitTags):
2088         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2089         if len(validLabelRegexp) == 0:
2090             validLabelRegexp = defaultLabelRegexp
2091         m = re.compile(validLabelRegexp)
2092
2093         for name in gitTags:
2094
2095             if not m.match(name):
2096                 if verbose:
2097                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2098                 continue
2099
2100             # Get the p4 commit this corresponds to
2101             logMessage = extractLogMessageFromGitCommit(name)
2102             values = extractSettingsGitLog(logMessage)
2103
2104             if 'change' not in values:
2105                 # a tag pointing to something not sent to p4; ignore
2106                 if verbose:
2107                     print("git tag %s does not give a p4 commit" % name)
2108                 continue
2109             else:
2110                 changelist = values['change']
2111
2112             # Get the tag details.
2113             inHeader = True
2114             isAnnotated = False
2115             body = []
2116             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2117                 l = l.strip()
2118                 if inHeader:
2119                     if re.match(r'tag\s+', l):
2120                         isAnnotated = True
2121                     elif re.match(r'\s*$', l):
2122                         inHeader = False
2123                         continue
2124                 else:
2125                     body.append(l)
2126
2127             if not isAnnotated:
2128                 body = ["lightweight tag imported by git p4\n"]
2129
2130             # Create the label - use the same view as the client spec we are using
2131             clientSpec = getClientSpec()
2132
2133             labelTemplate  = "Label: %s\n" % name
2134             labelTemplate += "Description:\n"
2135             for b in body:
2136                 labelTemplate += "\t" + b + "\n"
2137             labelTemplate += "View:\n"
2138             for depot_side in clientSpec.mappings:
2139                 labelTemplate += "\t%s\n" % depot_side
2140
2141             if self.dry_run:
2142                 print("Would create p4 label %s for tag" % name)
2143             elif self.prepare_p4_only:
2144                 print("Not creating p4 label %s for tag due to option" \
2145                       " --prepare-p4-only" % name)
2146             else:
2147                 p4_write_pipe(["label", "-i"], labelTemplate)
2148
2149                 # Use the label
2150                 p4_system(["tag", "-l", name] +
2151                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2152
2153                 if verbose:
2154                     print("created p4 label for tag %s" % name)
2155
2156     def run(self, args):
2157         if len(args) == 0:
2158             self.master = currentGitBranch()
2159         elif len(args) == 1:
2160             self.master = args[0]
2161             if not branchExists(self.master):
2162                 die("Branch %s does not exist" % self.master)
2163         else:
2164             return False
2165
2166         for i in self.update_shelve:
2167             if i <= 0:
2168                 sys.exit("invalid changelist %d" % i)
2169
2170         if self.master:
2171             allowSubmit = gitConfig("git-p4.allowSubmit")
2172             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2173                 die("%s is not in git-p4.allowSubmit" % self.master)
2174
2175         [upstream, settings] = findUpstreamBranchPoint()
2176         self.depotPath = settings['depot-paths'][0]
2177         if len(self.origin) == 0:
2178             self.origin = upstream
2179
2180         if len(self.update_shelve) > 0:
2181             self.shelve = True
2182
2183         if self.preserveUser:
2184             if not self.canChangeChangelists():
2185                 die("Cannot preserve user names without p4 super-user or admin permissions")
2186
2187         # if not set from the command line, try the config file
2188         if self.conflict_behavior is None:
2189             val = gitConfig("git-p4.conflict")
2190             if val:
2191                 if val not in self.conflict_behavior_choices:
2192                     die("Invalid value '%s' for config git-p4.conflict" % val)
2193             else:
2194                 val = "ask"
2195             self.conflict_behavior = val
2196
2197         if self.verbose:
2198             print("Origin branch is " + self.origin)
2199
2200         if len(self.depotPath) == 0:
2201             print("Internal error: cannot locate perforce depot path from existing branches")
2202             sys.exit(128)
2203
2204         self.useClientSpec = False
2205         if gitConfigBool("git-p4.useclientspec"):
2206             self.useClientSpec = True
2207         if self.useClientSpec:
2208             self.clientSpecDirs = getClientSpec()
2209
2210         # Check for the existence of P4 branches
2211         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2212
2213         if self.useClientSpec and not branchesDetected:
2214             # all files are relative to the client spec
2215             self.clientPath = getClientRoot()
2216         else:
2217             self.clientPath = p4Where(self.depotPath)
2218
2219         if self.clientPath == "":
2220             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2221
2222         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2223         self.oldWorkingDirectory = os.getcwd()
2224
2225         # ensure the clientPath exists
2226         new_client_dir = False
2227         if not os.path.exists(self.clientPath):
2228             new_client_dir = True
2229             os.makedirs(self.clientPath)
2230
2231         chdir(self.clientPath, is_client_path=True)
2232         if self.dry_run:
2233             print("Would synchronize p4 checkout in %s" % self.clientPath)
2234         else:
2235             print("Synchronizing p4 checkout...")
2236             if new_client_dir:
2237                 # old one was destroyed, and maybe nobody told p4
2238                 p4_sync("...", "-f")
2239             else:
2240                 p4_sync("...")
2241         self.check()
2242
2243         commits = []
2244         if self.master:
2245             committish = self.master
2246         else:
2247             committish = 'HEAD'
2248
2249         if self.commit != "":
2250             if self.commit.find("..") != -1:
2251                 limits_ish = self.commit.split("..")
2252                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2253                     commits.append(line.strip())
2254                 commits.reverse()
2255             else:
2256                 commits.append(self.commit)
2257         else:
2258             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2259                 commits.append(line.strip())
2260             commits.reverse()
2261
2262         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2263             self.checkAuthorship = False
2264         else:
2265             self.checkAuthorship = True
2266
2267         if self.preserveUser:
2268             self.checkValidP4Users(commits)
2269
2270         #
2271         # Build up a set of options to be passed to diff when
2272         # submitting each commit to p4.
2273         #
2274         if self.detectRenames:
2275             # command-line -M arg
2276             self.diffOpts = "-M"
2277         else:
2278             # If not explicitly set check the config variable
2279             detectRenames = gitConfig("git-p4.detectRenames")
2280
2281             if detectRenames.lower() == "false" or detectRenames == "":
2282                 self.diffOpts = ""
2283             elif detectRenames.lower() == "true":
2284                 self.diffOpts = "-M"
2285             else:
2286                 self.diffOpts = "-M%s" % detectRenames
2287
2288         # no command-line arg for -C or --find-copies-harder, just
2289         # config variables
2290         detectCopies = gitConfig("git-p4.detectCopies")
2291         if detectCopies.lower() == "false" or detectCopies == "":
2292             pass
2293         elif detectCopies.lower() == "true":
2294             self.diffOpts += " -C"
2295         else:
2296             self.diffOpts += " -C%s" % detectCopies
2297
2298         if gitConfigBool("git-p4.detectCopiesHarder"):
2299             self.diffOpts += " --find-copies-harder"
2300
2301         num_shelves = len(self.update_shelve)
2302         if num_shelves > 0 and num_shelves != len(commits):
2303             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2304                      (len(commits), num_shelves))
2305
2306         #
2307         # Apply the commits, one at a time.  On failure, ask if should
2308         # continue to try the rest of the patches, or quit.
2309         #
2310         if self.dry_run:
2311             print("Would apply")
2312         applied = []
2313         last = len(commits) - 1
2314         for i, commit in enumerate(commits):
2315             if self.dry_run:
2316                 print(" ", read_pipe(["git", "show", "-s",
2317                                       "--format=format:%h %s", commit]))
2318                 ok = True
2319             else:
2320                 ok = self.applyCommit(commit)
2321             if ok:
2322                 applied.append(commit)
2323             else:
2324                 if self.prepare_p4_only and i < last:
2325                     print("Processing only the first commit due to option" \
2326                           " --prepare-p4-only")
2327                     break
2328                 if i < last:
2329                     quit = False
2330                     while True:
2331                         # prompt for what to do, or use the option/variable
2332                         if self.conflict_behavior == "ask":
2333                             print("What do you want to do?")
2334                             response = raw_input("[s]kip this commit but apply"
2335                                                  " the rest, or [q]uit? ")
2336                             if not response:
2337                                 continue
2338                         elif self.conflict_behavior == "skip":
2339                             response = "s"
2340                         elif self.conflict_behavior == "quit":
2341                             response = "q"
2342                         else:
2343                             die("Unknown conflict_behavior '%s'" %
2344                                 self.conflict_behavior)
2345
2346                         if response[0] == "s":
2347                             print("Skipping this commit, but applying the rest")
2348                             break
2349                         if response[0] == "q":
2350                             print("Quitting")
2351                             quit = True
2352                             break
2353                     if quit:
2354                         break
2355
2356         chdir(self.oldWorkingDirectory)
2357         shelved_applied = "shelved" if self.shelve else "applied"
2358         if self.dry_run:
2359             pass
2360         elif self.prepare_p4_only:
2361             pass
2362         elif len(commits) == len(applied):
2363             print("All commits {0}!".format(shelved_applied))
2364
2365             sync = P4Sync()
2366             if self.branch:
2367                 sync.branch = self.branch
2368             if self.disable_p4sync:
2369                 sync.sync_origin_only()
2370             else:
2371                 sync.run([])
2372
2373                 if not self.disable_rebase:
2374                     rebase = P4Rebase()
2375                     rebase.rebase()
2376
2377         else:
2378             if len(applied) == 0:
2379                 print("No commits {0}.".format(shelved_applied))
2380             else:
2381                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2382                 for c in commits:
2383                     if c in applied:
2384                         star = "*"
2385                     else:
2386                         star = " "
2387                     print(star, read_pipe(["git", "show", "-s",
2388                                            "--format=format:%h %s",  c]))
2389                 print("You will have to do 'git p4 sync' and rebase.")
2390
2391         if gitConfigBool("git-p4.exportLabels"):
2392             self.exportLabels = True
2393
2394         if self.exportLabels:
2395             p4Labels = getP4Labels(self.depotPath)
2396             gitTags = getGitTags()
2397
2398             missingGitTags = gitTags - p4Labels
2399             self.exportGitTags(missingGitTags)
2400
2401         # exit with error unless everything applied perfectly
2402         if len(commits) != len(applied):
2403                 sys.exit(1)
2404
2405         return True
2406
2407 class View(object):
2408     """Represent a p4 view ("p4 help views"), and map files in a
2409        repo according to the view."""
2410
2411     def __init__(self, client_name):
2412         self.mappings = []
2413         self.client_prefix = "//%s/" % client_name
2414         # cache results of "p4 where" to lookup client file locations
2415         self.client_spec_path_cache = {}
2416
2417     def append(self, view_line):
2418         """Parse a view line, splitting it into depot and client
2419            sides.  Append to self.mappings, preserving order.  This
2420            is only needed for tag creation."""
2421
2422         # Split the view line into exactly two words.  P4 enforces
2423         # structure on these lines that simplifies this quite a bit.
2424         #
2425         # Either or both words may be double-quoted.
2426         # Single quotes do not matter.
2427         # Double-quote marks cannot occur inside the words.
2428         # A + or - prefix is also inside the quotes.
2429         # There are no quotes unless they contain a space.
2430         # The line is already white-space stripped.
2431         # The two words are separated by a single space.
2432         #
2433         if view_line[0] == '"':
2434             # First word is double quoted.  Find its end.
2435             close_quote_index = view_line.find('"', 1)
2436             if close_quote_index <= 0:
2437                 die("No first-word closing quote found: %s" % view_line)
2438             depot_side = view_line[1:close_quote_index]
2439             # skip closing quote and space
2440             rhs_index = close_quote_index + 1 + 1
2441         else:
2442             space_index = view_line.find(" ")
2443             if space_index <= 0:
2444                 die("No word-splitting space found: %s" % view_line)
2445             depot_side = view_line[0:space_index]
2446             rhs_index = space_index + 1
2447
2448         # prefix + means overlay on previous mapping
2449         if depot_side.startswith("+"):
2450             depot_side = depot_side[1:]
2451
2452         # prefix - means exclude this path, leave out of mappings
2453         exclude = False
2454         if depot_side.startswith("-"):
2455             exclude = True
2456             depot_side = depot_side[1:]
2457
2458         if not exclude:
2459             self.mappings.append(depot_side)
2460
2461     def convert_client_path(self, clientFile):
2462         # chop off //client/ part to make it relative
2463         if not clientFile.startswith(self.client_prefix):
2464             die("No prefix '%s' on clientFile '%s'" %
2465                 (self.client_prefix, clientFile))
2466         return clientFile[len(self.client_prefix):]
2467
2468     def update_client_spec_path_cache(self, files):
2469         """ Caching file paths by "p4 where" batch query """
2470
2471         # List depot file paths exclude that already cached
2472         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2473
2474         if len(fileArgs) == 0:
2475             return  # All files in cache
2476
2477         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2478         for res in where_result:
2479             if "code" in res and res["code"] == "error":
2480                 # assume error is "... file(s) not in client view"
2481                 continue
2482             if "clientFile" not in res:
2483                 die("No clientFile in 'p4 where' output")
2484             if "unmap" in res:
2485                 # it will list all of them, but only one not unmap-ped
2486                 continue
2487             if gitConfigBool("core.ignorecase"):
2488                 res['depotFile'] = res['depotFile'].lower()
2489             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2490
2491         # not found files or unmap files set to ""
2492         for depotFile in fileArgs:
2493             if gitConfigBool("core.ignorecase"):
2494                 depotFile = depotFile.lower()
2495             if depotFile not in self.client_spec_path_cache:
2496                 self.client_spec_path_cache[depotFile] = ""
2497
2498     def map_in_client(self, depot_path):
2499         """Return the relative location in the client where this
2500            depot file should live.  Returns "" if the file should
2501            not be mapped in the client."""
2502
2503         if gitConfigBool("core.ignorecase"):
2504             depot_path = depot_path.lower()
2505
2506         if depot_path in self.client_spec_path_cache:
2507             return self.client_spec_path_cache[depot_path]
2508
2509         die( "Error: %s is not found in client spec path" % depot_path )
2510         return ""
2511
2512 class P4Sync(Command, P4UserMap):
2513     delete_actions = ( "delete", "move/delete", "purge" )
2514
2515     def __init__(self):
2516         Command.__init__(self)
2517         P4UserMap.__init__(self)
2518         self.options = [
2519                 optparse.make_option("--branch", dest="branch"),
2520                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2521                 optparse.make_option("--changesfile", dest="changesFile"),
2522                 optparse.make_option("--silent", dest="silent", action="store_true"),
2523                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2524                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2525                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2526                                      help="Import into refs/heads/ , not refs/remotes"),
2527                 optparse.make_option("--max-changes", dest="maxChanges",
2528                                      help="Maximum number of changes to import"),
2529                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2530                                      help="Internal block size to use when iteratively calling p4 changes"),
2531                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2532                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2533                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2534                                      help="Only sync files that are included in the Perforce Client Spec"),
2535                 optparse.make_option("-/", dest="cloneExclude",
2536                                      action="append", type="string",
2537                                      help="exclude depot path"),
2538         ]
2539         self.description = """Imports from Perforce into a git repository.\n
2540     example:
2541     //depot/my/project/ -- to import the current head
2542     //depot/my/project/@all -- to import everything
2543     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2544
2545     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2546
2547         self.usage += " //depot/path[@revRange]"
2548         self.silent = False
2549         self.createdBranches = set()
2550         self.committedChanges = set()
2551         self.branch = ""
2552         self.detectBranches = False
2553         self.detectLabels = False
2554         self.importLabels = False
2555         self.changesFile = ""
2556         self.syncWithOrigin = True
2557         self.importIntoRemotes = True
2558         self.maxChanges = ""
2559         self.changes_block_size = None
2560         self.keepRepoPath = False
2561         self.depotPaths = None
2562         self.p4BranchesInGit = []
2563         self.cloneExclude = []
2564         self.useClientSpec = False
2565         self.useClientSpec_from_options = False
2566         self.clientSpecDirs = None
2567         self.tempBranches = []
2568         self.tempBranchLocation = "refs/git-p4-tmp"
2569         self.largeFileSystem = None
2570         self.suppress_meta_comment = False
2571
2572         if gitConfig('git-p4.largeFileSystem'):
2573             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2574             self.largeFileSystem = largeFileSystemConstructor(
2575                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2576             )
2577
2578         if gitConfig("git-p4.syncFromOrigin") == "false":
2579             self.syncWithOrigin = False
2580
2581         self.depotPaths = []
2582         self.changeRange = ""
2583         self.previousDepotPaths = []
2584         self.hasOrigin = False
2585
2586         # map from branch depot path to parent branch
2587         self.knownBranches = {}
2588         self.initialParents = {}
2589
2590         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2591         self.labels = {}
2592
2593     # Force a checkpoint in fast-import and wait for it to finish
2594     def checkpoint(self):
2595         self.gitStream.write("checkpoint\n\n")
2596         self.gitStream.write("progress checkpoint\n\n")
2597         out = self.gitOutput.readline()
2598         if self.verbose:
2599             print("checkpoint finished: " + out)
2600
2601     def cmp_shelved(self, path, filerev, revision):
2602         """ Determine if a path at revision #filerev is the same as the file
2603             at revision @revision for a shelved changelist. If they don't match,
2604             unshelving won't be safe (we will get other changes mixed in).
2605
2606             This is comparing the revision that the shelved changelist is *based* on, not
2607             the shelved changelist itself.
2608         """
2609         ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2610         if verbose:
2611             print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2612         return ret["status"] == "identical"
2613
2614     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
2615         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2616                              for path in self.cloneExclude]
2617         files = []
2618         fnum = 0
2619         while "depotFile%s" % fnum in commit:
2620             path =  commit["depotFile%s" % fnum]
2621
2622             if [p for p in self.cloneExclude
2623                 if p4PathStartsWith(path, p)]:
2624                 found = False
2625             else:
2626                 found = [p for p in self.depotPaths
2627                          if p4PathStartsWith(path, p)]
2628             if not found:
2629                 fnum = fnum + 1
2630                 continue
2631
2632             file = {}
2633             file["path"] = path
2634             file["rev"] = commit["rev%s" % fnum]
2635             file["action"] = commit["action%s" % fnum]
2636             file["type"] = commit["type%s" % fnum]
2637             if shelved:
2638                 file["shelved_cl"] = int(shelved_cl)
2639
2640                 # For shelved changelists, check that the revision of each file that the
2641                 # shelve was based on matches the revision that we are using for the
2642                 # starting point for git-fast-import (self.initialParent). Otherwise
2643                 # the resulting diff will contain deltas from multiple commits.
2644
2645                 if file["action"] != "add" and \
2646                     not self.cmp_shelved(path, file["rev"], origin_revision):
2647                     sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2648                         commit["change"], self.initialParent, path))
2649
2650             files.append(file)
2651             fnum = fnum + 1
2652         return files
2653
2654     def extractJobsFromCommit(self, commit):
2655         jobs = []
2656         jnum = 0
2657         while "job%s" % jnum in commit:
2658             job = commit["job%s" % jnum]
2659             jobs.append(job)
2660             jnum = jnum + 1
2661         return jobs
2662
2663     def stripRepoPath(self, path, prefixes):
2664         """When streaming files, this is called to map a p4 depot path
2665            to where it should go in git.  The prefixes are either
2666            self.depotPaths, or self.branchPrefixes in the case of
2667            branch detection."""
2668
2669         if self.useClientSpec:
2670             # branch detection moves files up a level (the branch name)
2671             # from what client spec interpretation gives
2672             path = self.clientSpecDirs.map_in_client(path)
2673             if self.detectBranches:
2674                 for b in self.knownBranches:
2675                     if path.startswith(b + "/"):
2676                         path = path[len(b)+1:]
2677
2678         elif self.keepRepoPath:
2679             # Preserve everything in relative path name except leading
2680             # //depot/; just look at first prefix as they all should
2681             # be in the same depot.
2682             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2683             if p4PathStartsWith(path, depot):
2684                 path = path[len(depot):]
2685
2686         else:
2687             for p in prefixes:
2688                 if p4PathStartsWith(path, p):
2689                     path = path[len(p):]
2690                     break
2691
2692         path = wildcard_decode(path)
2693         return path
2694
2695     def splitFilesIntoBranches(self, commit):
2696         """Look at each depotFile in the commit to figure out to what
2697            branch it belongs."""
2698
2699         if self.clientSpecDirs:
2700             files = self.extractFilesFromCommit(commit)
2701             self.clientSpecDirs.update_client_spec_path_cache(files)
2702
2703         branches = {}
2704         fnum = 0
2705         while "depotFile%s" % fnum in commit:
2706             path =  commit["depotFile%s" % fnum]
2707             found = [p for p in self.depotPaths
2708                      if p4PathStartsWith(path, p)]
2709             if not found:
2710                 fnum = fnum + 1
2711                 continue
2712
2713             file = {}
2714             file["path"] = path
2715             file["rev"] = commit["rev%s" % fnum]
2716             file["action"] = commit["action%s" % fnum]
2717             file["type"] = commit["type%s" % fnum]
2718             fnum = fnum + 1
2719
2720             # start with the full relative path where this file would
2721             # go in a p4 client
2722             if self.useClientSpec:
2723                 relPath = self.clientSpecDirs.map_in_client(path)
2724             else:
2725                 relPath = self.stripRepoPath(path, self.depotPaths)
2726
2727             for branch in self.knownBranches.keys():
2728                 # add a trailing slash so that a commit into qt/4.2foo
2729                 # doesn't end up in qt/4.2, e.g.
2730                 if relPath.startswith(branch + "/"):
2731                     if branch not in branches:
2732                         branches[branch] = []
2733                     branches[branch].append(file)
2734                     break
2735
2736         return branches
2737
2738     def writeToGitStream(self, gitMode, relPath, contents):
2739         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2740         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2741         for d in contents:
2742             self.gitStream.write(d)
2743         self.gitStream.write('\n')
2744
2745     def encodeWithUTF8(self, path):
2746         try:
2747             path.decode('ascii')
2748         except:
2749             encoding = 'utf8'
2750             if gitConfig('git-p4.pathEncoding'):
2751                 encoding = gitConfig('git-p4.pathEncoding')
2752             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2753             if self.verbose:
2754                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2755         return path
2756
2757     # output one file from the P4 stream
2758     # - helper for streamP4Files
2759
2760     def streamOneP4File(self, file, contents):
2761         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2762         relPath = self.encodeWithUTF8(relPath)
2763         if verbose:
2764             size = int(self.stream_file['fileSize'])
2765             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2766             sys.stdout.flush()
2767
2768         (type_base, type_mods) = split_p4_type(file["type"])
2769
2770         git_mode = "100644"
2771         if "x" in type_mods:
2772             git_mode = "100755"
2773         if type_base == "symlink":
2774             git_mode = "120000"
2775             # p4 print on a symlink sometimes contains "target\n";
2776             # if it does, remove the newline
2777             data = ''.join(contents)
2778             if not data:
2779                 # Some version of p4 allowed creating a symlink that pointed
2780                 # to nothing.  This causes p4 errors when checking out such
2781                 # a change, and errors here too.  Work around it by ignoring
2782                 # the bad symlink; hopefully a future change fixes it.
2783                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2784                 return
2785             elif data[-1] == '\n':
2786                 contents = [data[:-1]]
2787             else:
2788                 contents = [data]
2789
2790         if type_base == "utf16":
2791             # p4 delivers different text in the python output to -G
2792             # than it does when using "print -o", or normal p4 client
2793             # operations.  utf16 is converted to ascii or utf8, perhaps.
2794             # But ascii text saved as -t utf16 is completely mangled.
2795             # Invoke print -o to get the real contents.
2796             #
2797             # On windows, the newlines will always be mangled by print, so put
2798             # them back too.  This is not needed to the cygwin windows version,
2799             # just the native "NT" type.
2800             #
2801             try:
2802                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2803             except Exception as e:
2804                 if 'Translation of file content failed' in str(e):
2805                     type_base = 'binary'
2806                 else:
2807                     raise e
2808             else:
2809                 if p4_version_string().find('/NT') >= 0:
2810                     text = text.replace('\r\n', '\n')
2811                 contents = [ text ]
2812
2813         if type_base == "apple":
2814             # Apple filetype files will be streamed as a concatenation of
2815             # its appledouble header and the contents.  This is useless
2816             # on both macs and non-macs.  If using "print -q -o xx", it
2817             # will create "xx" with the data, and "%xx" with the header.
2818             # This is also not very useful.
2819             #
2820             # Ideally, someday, this script can learn how to generate
2821             # appledouble files directly and import those to git, but
2822             # non-mac machines can never find a use for apple filetype.
2823             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2824             return
2825
2826         # Note that we do not try to de-mangle keywords on utf16 files,
2827         # even though in theory somebody may want that.
2828         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2829         if pattern:
2830             regexp = re.compile(pattern, re.VERBOSE)
2831             text = ''.join(contents)
2832             text = regexp.sub(r'$\1$', text)
2833             contents = [ text ]
2834
2835         if self.largeFileSystem:
2836             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2837
2838         self.writeToGitStream(git_mode, relPath, contents)
2839
2840     def streamOneP4Deletion(self, file):
2841         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2842         relPath = self.encodeWithUTF8(relPath)
2843         if verbose:
2844             sys.stdout.write("delete %s\n" % relPath)
2845             sys.stdout.flush()
2846         self.gitStream.write("D %s\n" % relPath)
2847
2848         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2849             self.largeFileSystem.removeLargeFile(relPath)
2850
2851     # handle another chunk of streaming data
2852     def streamP4FilesCb(self, marshalled):
2853
2854         # catch p4 errors and complain
2855         err = None
2856         if "code" in marshalled:
2857             if marshalled["code"] == "error":
2858                 if "data" in marshalled:
2859                     err = marshalled["data"].rstrip()
2860
2861         if not err and 'fileSize' in self.stream_file:
2862             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2863             if required_bytes > 0:
2864                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2865                     os.getcwd(), required_bytes/1024/1024
2866                 )
2867
2868         if err:
2869             f = None
2870             if self.stream_have_file_info:
2871                 if "depotFile" in self.stream_file:
2872                     f = self.stream_file["depotFile"]
2873             # force a failure in fast-import, else an empty
2874             # commit will be made
2875             self.gitStream.write("\n")
2876             self.gitStream.write("die-now\n")
2877             self.gitStream.close()
2878             # ignore errors, but make sure it exits first
2879             self.importProcess.wait()
2880             if f:
2881                 die("Error from p4 print for %s: %s" % (f, err))
2882             else:
2883                 die("Error from p4 print: %s" % err)
2884
2885         if 'depotFile' in marshalled and self.stream_have_file_info:
2886             # start of a new file - output the old one first
2887             self.streamOneP4File(self.stream_file, self.stream_contents)
2888             self.stream_file = {}
2889             self.stream_contents = []
2890             self.stream_have_file_info = False
2891
2892         # pick up the new file information... for the
2893         # 'data' field we need to append to our array
2894         for k in marshalled.keys():
2895             if k == 'data':
2896                 if 'streamContentSize' not in self.stream_file:
2897                     self.stream_file['streamContentSize'] = 0
2898                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2899                 self.stream_contents.append(marshalled['data'])
2900             else:
2901                 self.stream_file[k] = marshalled[k]
2902
2903         if (verbose and
2904             'streamContentSize' in self.stream_file and
2905             'fileSize' in self.stream_file and
2906             'depotFile' in self.stream_file):
2907             size = int(self.stream_file["fileSize"])
2908             if size > 0:
2909                 progress = 100*self.stream_file['streamContentSize']/size
2910                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2911                 sys.stdout.flush()
2912
2913         self.stream_have_file_info = True
2914
2915     # Stream directly from "p4 files" into "git fast-import"
2916     def streamP4Files(self, files):
2917         filesForCommit = []
2918         filesToRead = []
2919         filesToDelete = []
2920
2921         for f in files:
2922             filesForCommit.append(f)
2923             if f['action'] in self.delete_actions:
2924                 filesToDelete.append(f)
2925             else:
2926                 filesToRead.append(f)
2927
2928         # deleted files...
2929         for f in filesToDelete:
2930             self.streamOneP4Deletion(f)
2931
2932         if len(filesToRead) > 0:
2933             self.stream_file = {}
2934             self.stream_contents = []
2935             self.stream_have_file_info = False
2936
2937             # curry self argument
2938             def streamP4FilesCbSelf(entry):
2939                 self.streamP4FilesCb(entry)
2940
2941             fileArgs = []
2942             for f in filesToRead:
2943                 if 'shelved_cl' in f:
2944                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2945                     # the contents
2946                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2947                 else:
2948                     fileArg = '%s#%s' % (f['path'], f['rev'])
2949
2950                 fileArgs.append(fileArg)
2951
2952             p4CmdList(["-x", "-", "print"],
2953                       stdin=fileArgs,
2954                       cb=streamP4FilesCbSelf)
2955
2956             # do the last chunk
2957             if 'depotFile' in self.stream_file:
2958                 self.streamOneP4File(self.stream_file, self.stream_contents)
2959
2960     def make_email(self, userid):
2961         if userid in self.users:
2962             return self.users[userid]
2963         else:
2964             return "%s <a@b>" % userid
2965
2966     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2967         """ Stream a p4 tag.
2968         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2969         """
2970
2971         if verbose:
2972             print("writing tag %s for commit %s" % (labelName, commit))
2973         gitStream.write("tag %s\n" % labelName)
2974         gitStream.write("from %s\n" % commit)
2975
2976         if 'Owner' in labelDetails:
2977             owner = labelDetails["Owner"]
2978         else:
2979             owner = None
2980
2981         # Try to use the owner of the p4 label, or failing that,
2982         # the current p4 user id.
2983         if owner:
2984             email = self.make_email(owner)
2985         else:
2986             email = self.make_email(self.p4UserId())
2987         tagger = "%s %s %s" % (email, epoch, self.tz)
2988
2989         gitStream.write("tagger %s\n" % tagger)
2990
2991         print("labelDetails=",labelDetails)
2992         if 'Description' in labelDetails:
2993             description = labelDetails['Description']
2994         else:
2995             description = 'Label from git p4'
2996
2997         gitStream.write("data %d\n" % len(description))
2998         gitStream.write(description)
2999         gitStream.write("\n")
3000
3001     def inClientSpec(self, path):
3002         if not self.clientSpecDirs:
3003             return True
3004         inClientSpec = self.clientSpecDirs.map_in_client(path)
3005         if not inClientSpec and self.verbose:
3006             print('Ignoring file outside of client spec: {0}'.format(path))
3007         return inClientSpec
3008
3009     def hasBranchPrefix(self, path):
3010         if not self.branchPrefixes:
3011             return True
3012         hasPrefix = [p for p in self.branchPrefixes
3013                         if p4PathStartsWith(path, p)]
3014         if not hasPrefix and self.verbose:
3015             print('Ignoring file outside of prefix: {0}'.format(path))
3016         return hasPrefix
3017
3018     def commit(self, details, files, branch, parent = ""):
3019         epoch = details["time"]
3020         author = details["user"]
3021         jobs = self.extractJobsFromCommit(details)
3022
3023         if self.verbose:
3024             print('commit into {0}'.format(branch))
3025
3026         if self.clientSpecDirs:
3027             self.clientSpecDirs.update_client_spec_path_cache(files)
3028
3029         files = [f for f in files
3030             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3031
3032         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
3033             print('Ignoring revision {0} as it would produce an empty commit.'
3034                 .format(details['change']))
3035             return
3036
3037         self.gitStream.write("commit %s\n" % branch)
3038         self.gitStream.write("mark :%s\n" % details["change"])
3039         self.committedChanges.add(int(details["change"]))
3040         committer = ""
3041         if author not in self.users:
3042             self.getUserMapFromPerforceServer()
3043         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3044
3045         self.gitStream.write("committer %s\n" % committer)
3046
3047         self.gitStream.write("data <<EOT\n")
3048         self.gitStream.write(details["desc"])
3049         if len(jobs) > 0:
3050             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3051
3052         if not self.suppress_meta_comment:
3053             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3054                                 (','.join(self.branchPrefixes), details["change"]))
3055             if len(details['options']) > 0:
3056                 self.gitStream.write(": options = %s" % details['options'])
3057             self.gitStream.write("]\n")
3058
3059         self.gitStream.write("EOT\n\n")
3060
3061         if len(parent) > 0:
3062             if self.verbose:
3063                 print("parent %s" % parent)
3064             self.gitStream.write("from %s\n" % parent)
3065
3066         self.streamP4Files(files)
3067         self.gitStream.write("\n")
3068
3069         change = int(details["change"])
3070
3071         if change in self.labels:
3072             label = self.labels[change]
3073             labelDetails = label[0]
3074             labelRevisions = label[1]
3075             if self.verbose:
3076                 print("Change %s is labelled %s" % (change, labelDetails))
3077
3078             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3079                                                 for p in self.branchPrefixes])
3080
3081             if len(files) == len(labelRevisions):
3082
3083                 cleanedFiles = {}
3084                 for info in files:
3085                     if info["action"] in self.delete_actions:
3086                         continue
3087                     cleanedFiles[info["depotFile"]] = info["rev"]
3088
3089                 if cleanedFiles == labelRevisions:
3090                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3091
3092                 else:
3093                     if not self.silent:
3094                         print("Tag %s does not match with change %s: files do not match."
3095                                % (labelDetails["label"], change))
3096
3097             else:
3098                 if not self.silent:
3099                     print("Tag %s does not match with change %s: file count is different."
3100                            % (labelDetails["label"], change))
3101
3102     # Build a dictionary of changelists and labels, for "detect-labels" option.
3103     def getLabels(self):
3104         self.labels = {}
3105
3106         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3107         if len(l) > 0 and not self.silent:
3108             print("Finding files belonging to labels in %s" % self.depotPaths)
3109
3110         for output in l:
3111             label = output["label"]
3112             revisions = {}
3113             newestChange = 0
3114             if self.verbose:
3115                 print("Querying files for label %s" % label)
3116             for file in p4CmdList(["files"] +
3117                                       ["%s...@%s" % (p, label)
3118                                           for p in self.depotPaths]):
3119                 revisions[file["depotFile"]] = file["rev"]
3120                 change = int(file["change"])
3121                 if change > newestChange:
3122                     newestChange = change
3123
3124             self.labels[newestChange] = [output, revisions]
3125
3126         if self.verbose:
3127             print("Label changes: %s" % self.labels.keys())
3128
3129     # Import p4 labels as git tags. A direct mapping does not
3130     # exist, so assume that if all the files are at the same revision
3131     # then we can use that, or it's something more complicated we should
3132     # just ignore.
3133     def importP4Labels(self, stream, p4Labels):
3134         if verbose:
3135             print("import p4 labels: " + ' '.join(p4Labels))
3136
3137         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3138         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3139         if len(validLabelRegexp) == 0:
3140             validLabelRegexp = defaultLabelRegexp
3141         m = re.compile(validLabelRegexp)
3142
3143         for name in p4Labels:
3144             commitFound = False
3145
3146             if not m.match(name):
3147                 if verbose:
3148                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3149                 continue
3150
3151             if name in ignoredP4Labels:
3152                 continue
3153
3154             labelDetails = p4CmdList(['label', "-o", name])[0]
3155
3156             # get the most recent changelist for each file in this label
3157             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3158                                 for p in self.depotPaths])
3159
3160             if 'change' in change:
3161                 # find the corresponding git commit; take the oldest commit
3162                 changelist = int(change['change'])
3163                 if changelist in self.committedChanges:
3164                     gitCommit = ":%d" % changelist       # use a fast-import mark
3165                     commitFound = True
3166                 else:
3167                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3168                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3169                     if len(gitCommit) == 0:
3170                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3171                     else:
3172                         commitFound = True
3173                         gitCommit = gitCommit.strip()
3174
3175                 if commitFound:
3176                     # Convert from p4 time format
3177                     try:
3178                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3179                     except ValueError:
3180                         print("Could not convert label time %s" % labelDetails['Update'])
3181                         tmwhen = 1
3182
3183                     when = int(time.mktime(tmwhen))
3184                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3185                     if verbose:
3186                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3187             else:
3188                 if verbose:
3189                     print("Label %s has no changelists - possibly deleted?" % name)
3190
3191             if not commitFound:
3192                 # We can't import this label; don't try again as it will get very
3193                 # expensive repeatedly fetching all the files for labels that will
3194                 # never be imported. If the label is moved in the future, the
3195                 # ignore will need to be removed manually.
3196                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3197
3198     def guessProjectName(self):
3199         for p in self.depotPaths:
3200             if p.endswith("/"):
3201                 p = p[:-1]
3202             p = p[p.strip().rfind("/") + 1:]
3203             if not p.endswith("/"):
3204                p += "/"
3205             return p
3206
3207     def getBranchMapping(self):
3208         lostAndFoundBranches = set()
3209
3210         user = gitConfig("git-p4.branchUser")
3211         if len(user) > 0:
3212             command = "branches -u %s" % user
3213         else:
3214             command = "branches"
3215
3216         for info in p4CmdList(command):
3217             details = p4Cmd(["branch", "-o", info["branch"]])
3218             viewIdx = 0
3219             while "View%s" % viewIdx in details:
3220                 paths = details["View%s" % viewIdx].split(" ")
3221                 viewIdx = viewIdx + 1
3222                 # require standard //depot/foo/... //depot/bar/... mapping
3223                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3224                     continue
3225                 source = paths[0]
3226                 destination = paths[1]
3227                 ## HACK
3228                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3229                     source = source[len(self.depotPaths[0]):-4]
3230                     destination = destination[len(self.depotPaths[0]):-4]
3231
3232                     if destination in self.knownBranches:
3233                         if not self.silent:
3234                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3235                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3236                         continue
3237
3238                     self.knownBranches[destination] = source
3239
3240                     lostAndFoundBranches.discard(destination)
3241
3242                     if source not in self.knownBranches:
3243                         lostAndFoundBranches.add(source)
3244
3245         # Perforce does not strictly require branches to be defined, so we also
3246         # check git config for a branch list.
3247         #
3248         # Example of branch definition in git config file:
3249         # [git-p4]
3250         #   branchList=main:branchA
3251         #   branchList=main:branchB
3252         #   branchList=branchA:branchC
3253         configBranches = gitConfigList("git-p4.branchList")
3254         for branch in configBranches:
3255             if branch:
3256                 (source, destination) = branch.split(":")
3257                 self.knownBranches[destination] = source
3258
3259                 lostAndFoundBranches.discard(destination)
3260
3261                 if source not in self.knownBranches:
3262                     lostAndFoundBranches.add(source)
3263
3264
3265         for branch in lostAndFoundBranches:
3266             self.knownBranches[branch] = branch
3267
3268     def getBranchMappingFromGitBranches(self):
3269         branches = p4BranchesInGit(self.importIntoRemotes)
3270         for branch in branches.keys():
3271             if branch == "master":
3272                 branch = "main"
3273             else:
3274                 branch = branch[len(self.projectName):]
3275             self.knownBranches[branch] = branch
3276
3277     def updateOptionDict(self, d):
3278         option_keys = {}
3279         if self.keepRepoPath:
3280             option_keys['keepRepoPath'] = 1
3281
3282         d["options"] = ' '.join(sorted(option_keys.keys()))
3283
3284     def readOptions(self, d):
3285         self.keepRepoPath = ('options' in d
3286                              and ('keepRepoPath' in d['options']))
3287
3288     def gitRefForBranch(self, branch):
3289         if branch == "main":
3290             return self.refPrefix + "master"
3291
3292         if len(branch) <= 0:
3293             return branch
3294
3295         return self.refPrefix + self.projectName + branch
3296
3297     def gitCommitByP4Change(self, ref, change):
3298         if self.verbose:
3299             print("looking in ref " + ref + " for change %s using bisect..." % change)
3300
3301         earliestCommit = ""
3302         latestCommit = parseRevision(ref)
3303
3304         while True:
3305             if self.verbose:
3306                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3307             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3308             if len(next) == 0:
3309                 if self.verbose:
3310                     print("argh")
3311                 return ""
3312             log = extractLogMessageFromGitCommit(next)
3313             settings = extractSettingsGitLog(log)
3314             currentChange = int(settings['change'])
3315             if self.verbose:
3316                 print("current change %s" % currentChange)
3317
3318             if currentChange == change:
3319                 if self.verbose:
3320                     print("found %s" % next)
3321                 return next
3322
3323             if currentChange < change:
3324                 earliestCommit = "^%s" % next
3325             else:
3326                 latestCommit = "%s" % next
3327
3328         return ""
3329
3330     def importNewBranch(self, branch, maxChange):
3331         # make fast-import flush all changes to disk and update the refs using the checkpoint
3332         # command so that we can try to find the branch parent in the git history
3333         self.gitStream.write("checkpoint\n\n");
3334         self.gitStream.flush();
3335         branchPrefix = self.depotPaths[0] + branch + "/"
3336         range = "@1,%s" % maxChange
3337         #print "prefix" + branchPrefix
3338         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3339         if len(changes) <= 0:
3340             return False
3341         firstChange = changes[0]
3342         #print "first change in branch: %s" % firstChange
3343         sourceBranch = self.knownBranches[branch]
3344         sourceDepotPath = self.depotPaths[0] + sourceBranch
3345         sourceRef = self.gitRefForBranch(sourceBranch)
3346         #print "source " + sourceBranch
3347
3348         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3349         #print "branch parent: %s" % branchParentChange
3350         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3351         if len(gitParent) > 0:
3352             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3353             #print "parent git commit: %s" % gitParent
3354
3355         self.importChanges(changes)
3356         return True
3357
3358     def searchParent(self, parent, branch, target):
3359         parentFound = False
3360         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3361                                      "--no-merges", parent]):
3362             blob = blob.strip()
3363             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3364                 parentFound = True
3365                 if self.verbose:
3366                     print("Found parent of %s in commit %s" % (branch, blob))
3367                 break
3368         if parentFound:
3369             return blob
3370         else:
3371             return None
3372
3373     def importChanges(self, changes, shelved=False, origin_revision=0):
3374         cnt = 1
3375         for change in changes:
3376             description = p4_describe(change, shelved)
3377             self.updateOptionDict(description)
3378
3379             if not self.silent:
3380                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3381                 sys.stdout.flush()
3382             cnt = cnt + 1
3383
3384             try:
3385                 if self.detectBranches:
3386                     branches = self.splitFilesIntoBranches(description)
3387                     for branch in branches.keys():
3388                         ## HACK  --hwn
3389                         branchPrefix = self.depotPaths[0] + branch + "/"
3390                         self.branchPrefixes = [ branchPrefix ]
3391
3392                         parent = ""
3393
3394                         filesForCommit = branches[branch]
3395
3396                         if self.verbose:
3397                             print("branch is %s" % branch)
3398
3399                         self.updatedBranches.add(branch)
3400
3401                         if branch not in self.createdBranches:
3402                             self.createdBranches.add(branch)
3403                             parent = self.knownBranches[branch]
3404                             if parent == branch:
3405                                 parent = ""
3406                             else:
3407                                 fullBranch = self.projectName + branch
3408                                 if fullBranch not in self.p4BranchesInGit:
3409                                     if not self.silent:
3410                                         print("\n    Importing new branch %s" % fullBranch);
3411                                     if self.importNewBranch(branch, change - 1):
3412                                         parent = ""
3413                                         self.p4BranchesInGit.append(fullBranch)
3414                                     if not self.silent:
3415                                         print("\n    Resuming with change %s" % change);
3416
3417                                 if self.verbose:
3418                                     print("parent determined through known branches: %s" % parent)
3419
3420                         branch = self.gitRefForBranch(branch)
3421                         parent = self.gitRefForBranch(parent)
3422
3423                         if self.verbose:
3424                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3425
3426                         if len(parent) == 0 and branch in self.initialParents:
3427                             parent = self.initialParents[branch]
3428                             del self.initialParents[branch]
3429
3430                         blob = None
3431                         if len(parent) > 0:
3432                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3433                             if self.verbose:
3434                                 print("Creating temporary branch: " + tempBranch)
3435                             self.commit(description, filesForCommit, tempBranch)
3436                             self.tempBranches.append(tempBranch)
3437                             self.checkpoint()
3438                             blob = self.searchParent(parent, branch, tempBranch)
3439                         if blob:
3440                             self.commit(description, filesForCommit, branch, blob)
3441                         else:
3442                             if self.verbose:
3443                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3444                             self.commit(description, filesForCommit, branch, parent)
3445                 else:
3446                     files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
3447                     self.commit(description, files, self.branch,
3448                                 self.initialParent)
3449                     # only needed once, to connect to the previous commit
3450                     self.initialParent = ""
3451             except IOError:
3452                 print(self.gitError.read())
3453                 sys.exit(1)
3454
3455     def sync_origin_only(self):
3456         if self.syncWithOrigin:
3457             self.hasOrigin = originP4BranchesExist()
3458             if self.hasOrigin:
3459                 if not self.silent:
3460                     print('Syncing with origin first, using "git fetch origin"')
3461                 system("git fetch origin")
3462
3463     def importHeadRevision(self, revision):
3464         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3465
3466         details = {}
3467         details["user"] = "git perforce import user"
3468         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3469                            % (' '.join(self.depotPaths), revision))
3470         details["change"] = revision
3471         newestRevision = 0
3472
3473         fileCnt = 0
3474         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3475
3476         for info in p4CmdList(["files"] + fileArgs):
3477
3478             if 'code' in info and info['code'] == 'error':
3479                 sys.stderr.write("p4 returned an error: %s\n"
3480                                  % info['data'])
3481                 if info['data'].find("must refer to client") >= 0:
3482                     sys.stderr.write("This particular p4 error is misleading.\n")
3483                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3484                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3485                 sys.exit(1)
3486             if 'p4ExitCode' in info:
3487                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3488                 sys.exit(1)
3489
3490
3491             change = int(info["change"])
3492             if change > newestRevision:
3493                 newestRevision = change
3494
3495             if info["action"] in self.delete_actions:
3496                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3497                 #fileCnt = fileCnt + 1
3498                 continue
3499
3500             for prop in ["depotFile", "rev", "action", "type" ]:
3501                 details["%s%s" % (prop, fileCnt)] = info[prop]
3502
3503             fileCnt = fileCnt + 1
3504
3505         details["change"] = newestRevision
3506
3507         # Use time from top-most change so that all git p4 clones of
3508         # the same p4 repo have the same commit SHA1s.
3509         res = p4_describe(newestRevision)
3510         details["time"] = res["time"]
3511
3512         self.updateOptionDict(details)
3513         try:
3514             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3515         except IOError:
3516             print("IO error with git fast-import. Is your git version recent enough?")
3517             print(self.gitError.read())
3518
3519     def openStreams(self):
3520         self.importProcess = subprocess.Popen(["git", "fast-import"],
3521                                               stdin=subprocess.PIPE,
3522                                               stdout=subprocess.PIPE,
3523                                               stderr=subprocess.PIPE);
3524         self.gitOutput = self.importProcess.stdout
3525         self.gitStream = self.importProcess.stdin
3526         self.gitError = self.importProcess.stderr
3527
3528     def closeStreams(self):
3529         self.gitStream.close()
3530         if self.importProcess.wait() != 0:
3531             die("fast-import failed: %s" % self.gitError.read())
3532         self.gitOutput.close()
3533         self.gitError.close()
3534
3535     def run(self, args):
3536         if self.importIntoRemotes:
3537             self.refPrefix = "refs/remotes/p4/"
3538         else:
3539             self.refPrefix = "refs/heads/p4/"
3540
3541         self.sync_origin_only()
3542
3543         branch_arg_given = bool(self.branch)
3544         if len(self.branch) == 0:
3545             self.branch = self.refPrefix + "master"
3546             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3547                 system("git update-ref %s refs/heads/p4" % self.branch)
3548                 system("git branch -D p4")
3549
3550         # accept either the command-line option, or the configuration variable
3551         if self.useClientSpec:
3552             # will use this after clone to set the variable
3553             self.useClientSpec_from_options = True
3554         else:
3555             if gitConfigBool("git-p4.useclientspec"):
3556                 self.useClientSpec = True
3557         if self.useClientSpec:
3558             self.clientSpecDirs = getClientSpec()
3559
3560         # TODO: should always look at previous commits,
3561         # merge with previous imports, if possible.
3562         if args == []:
3563             if self.hasOrigin:
3564                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3565
3566             # branches holds mapping from branch name to sha1
3567             branches = p4BranchesInGit(self.importIntoRemotes)
3568
3569             # restrict to just this one, disabling detect-branches
3570             if branch_arg_given:
3571                 short = self.branch.split("/")[-1]
3572                 if short in branches:
3573                     self.p4BranchesInGit = [ short ]
3574             else:
3575                 self.p4BranchesInGit = branches.keys()
3576
3577             if len(self.p4BranchesInGit) > 1:
3578                 if not self.silent:
3579                     print("Importing from/into multiple branches")
3580                 self.detectBranches = True
3581                 for branch in branches.keys():
3582                     self.initialParents[self.refPrefix + branch] = \
3583                         branches[branch]
3584
3585             if self.verbose:
3586                 print("branches: %s" % self.p4BranchesInGit)
3587
3588             p4Change = 0
3589             for branch in self.p4BranchesInGit:
3590                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3591
3592                 settings = extractSettingsGitLog(logMsg)
3593
3594                 self.readOptions(settings)
3595                 if ('depot-paths' in settings
3596                     and 'change' in settings):
3597                     change = int(settings['change']) + 1
3598                     p4Change = max(p4Change, change)
3599
3600                     depotPaths = sorted(settings['depot-paths'])
3601                     if self.previousDepotPaths == []:
3602                         self.previousDepotPaths = depotPaths
3603                     else:
3604                         paths = []
3605                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3606                             prev_list = prev.split("/")
3607                             cur_list = cur.split("/")
3608                             for i in range(0, min(len(cur_list), len(prev_list))):
3609                                 if cur_list[i] != prev_list[i]:
3610                                     i = i - 1
3611                                     break
3612
3613                             paths.append ("/".join(cur_list[:i + 1]))
3614
3615                         self.previousDepotPaths = paths
3616
3617             if p4Change > 0:
3618                 self.depotPaths = sorted(self.previousDepotPaths)
3619                 self.changeRange = "@%s,#head" % p4Change
3620                 if not self.silent and not self.detectBranches:
3621                     print("Performing incremental import into %s git branch" % self.branch)
3622
3623         # accept multiple ref name abbreviations:
3624         #    refs/foo/bar/branch -> use it exactly
3625         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3626         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3627         if not self.branch.startswith("refs/"):
3628             if self.importIntoRemotes:
3629                 prepend = "refs/remotes/"
3630             else:
3631                 prepend = "refs/heads/"
3632             if not self.branch.startswith("p4/"):
3633                 prepend += "p4/"
3634             self.branch = prepend + self.branch
3635
3636         if len(args) == 0 and self.depotPaths:
3637             if not self.silent:
3638                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3639         else:
3640             if self.depotPaths and self.depotPaths != args:
3641                 print("previous import used depot path %s and now %s was specified. "
3642                        "This doesn't work!" % (' '.join (self.depotPaths),
3643                                                ' '.join (args)))
3644                 sys.exit(1)
3645
3646             self.depotPaths = sorted(args)
3647
3648         revision = ""
3649         self.users = {}
3650
3651         # Make sure no revision specifiers are used when --changesfile
3652         # is specified.
3653         bad_changesfile = False
3654         if len(self.changesFile) > 0:
3655             for p in self.depotPaths:
3656                 if p.find("@") >= 0 or p.find("#") >= 0:
3657                     bad_changesfile = True
3658                     break
3659         if bad_changesfile:
3660             die("Option --changesfile is incompatible with revision specifiers")
3661
3662         newPaths = []
3663         for p in self.depotPaths:
3664             if p.find("@") != -1:
3665                 atIdx = p.index("@")
3666                 self.changeRange = p[atIdx:]
3667                 if self.changeRange == "@all":
3668                     self.changeRange = ""
3669                 elif ',' not in self.changeRange:
3670                     revision = self.changeRange
3671                     self.changeRange = ""
3672                 p = p[:atIdx]
3673             elif p.find("#") != -1:
3674                 hashIdx = p.index("#")
3675                 revision = p[hashIdx:]
3676                 p = p[:hashIdx]
3677             elif self.previousDepotPaths == []:
3678                 # pay attention to changesfile, if given, else import
3679                 # the entire p4 tree at the head revision
3680                 if len(self.changesFile) == 0:
3681                     revision = "#head"
3682
3683             p = re.sub ("\.\.\.$", "", p)
3684             if not p.endswith("/"):
3685                 p += "/"
3686
3687             newPaths.append(p)
3688
3689         self.depotPaths = newPaths
3690
3691         # --detect-branches may change this for each branch
3692         self.branchPrefixes = self.depotPaths
3693
3694         self.loadUserMapFromCache()
3695         self.labels = {}
3696         if self.detectLabels:
3697             self.getLabels();
3698
3699         if self.detectBranches:
3700             ## FIXME - what's a P4 projectName ?
3701             self.projectName = self.guessProjectName()
3702
3703             if self.hasOrigin:
3704                 self.getBranchMappingFromGitBranches()
3705             else:
3706                 self.getBranchMapping()
3707             if self.verbose:
3708                 print("p4-git branches: %s" % self.p4BranchesInGit)
3709                 print("initial parents: %s" % self.initialParents)
3710             for b in self.p4BranchesInGit:
3711                 if b != "master":
3712
3713                     ## FIXME
3714                     b = b[len(self.projectName):]
3715                 self.createdBranches.add(b)
3716
3717         self.openStreams()
3718
3719         if revision:
3720             self.importHeadRevision(revision)
3721         else:
3722             changes = []
3723
3724             if len(self.changesFile) > 0:
3725                 output = open(self.changesFile).readlines()
3726                 changeSet = set()
3727                 for line in output:
3728                     changeSet.add(int(line))
3729
3730                 for change in changeSet:
3731                     changes.append(change)
3732
3733                 changes.sort()
3734             else:
3735                 # catch "git p4 sync" with no new branches, in a repo that
3736                 # does not have any existing p4 branches
3737                 if len(args) == 0:
3738                     if not self.p4BranchesInGit:
3739                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3740
3741                     # The default branch is master, unless --branch is used to
3742                     # specify something else.  Make sure it exists, or complain
3743                     # nicely about how to use --branch.
3744                     if not self.detectBranches:
3745                         if not branch_exists(self.branch):
3746                             if branch_arg_given:
3747                                 die("Error: branch %s does not exist." % self.branch)
3748                             else:
3749                                 die("Error: no branch %s; perhaps specify one with --branch." %
3750                                     self.branch)
3751
3752                 if self.verbose:
3753                     print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3754                                                               self.changeRange))
3755                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3756
3757                 if len(self.maxChanges) > 0:
3758                     changes = changes[:min(int(self.maxChanges), len(changes))]
3759
3760             if len(changes) == 0:
3761                 if not self.silent:
3762                     print("No changes to import!")
3763             else:
3764                 if not self.silent and not self.detectBranches:
3765                     print("Import destination: %s" % self.branch)
3766
3767                 self.updatedBranches = set()
3768
3769                 if not self.detectBranches:
3770                     if args:
3771                         # start a new branch
3772                         self.initialParent = ""
3773                     else:
3774                         # build on a previous revision
3775                         self.initialParent = parseRevision(self.branch)
3776
3777                 self.importChanges(changes)
3778
3779                 if not self.silent:
3780                     print("")
3781                     if len(self.updatedBranches) > 0:
3782                         sys.stdout.write("Updated branches: ")
3783                         for b in self.updatedBranches:
3784                             sys.stdout.write("%s " % b)
3785                         sys.stdout.write("\n")
3786
3787         if gitConfigBool("git-p4.importLabels"):
3788             self.importLabels = True
3789
3790         if self.importLabels:
3791             p4Labels = getP4Labels(self.depotPaths)
3792             gitTags = getGitTags()
3793
3794             missingP4Labels = p4Labels - gitTags
3795             self.importP4Labels(self.gitStream, missingP4Labels)
3796
3797         self.closeStreams()
3798
3799         # Cleanup temporary branches created during import
3800         if self.tempBranches != []:
3801             for branch in self.tempBranches:
3802                 read_pipe("git update-ref -d %s" % branch)
3803             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3804
3805         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3806         # a convenient shortcut refname "p4".
3807         if self.importIntoRemotes:
3808             head_ref = self.refPrefix + "HEAD"
3809             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3810                 system(["git", "symbolic-ref", head_ref, self.branch])
3811
3812         return True
3813
3814 class P4Rebase(Command):
3815     def __init__(self):
3816         Command.__init__(self)
3817         self.options = [
3818                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3819         ]
3820         self.importLabels = False
3821         self.description = ("Fetches the latest revision from perforce and "
3822                             + "rebases the current work (branch) against it")
3823
3824     def run(self, args):
3825         sync = P4Sync()
3826         sync.importLabels = self.importLabels
3827         sync.run([])
3828
3829         return self.rebase()
3830
3831     def rebase(self):
3832         if os.system("git update-index --refresh") != 0:
3833             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3834         if len(read_pipe("git diff-index HEAD --")) > 0:
3835             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3836
3837         [upstream, settings] = findUpstreamBranchPoint()
3838         if len(upstream) == 0:
3839             die("Cannot find upstream branchpoint for rebase")
3840
3841         # the branchpoint may be p4/foo~3, so strip off the parent
3842         upstream = re.sub("~[0-9]+$", "", upstream)
3843
3844         print("Rebasing the current branch onto %s" % upstream)
3845         oldHead = read_pipe("git rev-parse HEAD").strip()
3846         system("git rebase %s" % upstream)
3847         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3848         return True
3849
3850 class P4Clone(P4Sync):
3851     def __init__(self):
3852         P4Sync.__init__(self)
3853         self.description = "Creates a new git repository and imports from Perforce into it"
3854         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3855         self.options += [
3856             optparse.make_option("--destination", dest="cloneDestination",
3857                                  action='store', default=None,
3858                                  help="where to leave result of the clone"),
3859             optparse.make_option("--bare", dest="cloneBare",
3860                                  action="store_true", default=False),
3861         ]
3862         self.cloneDestination = None
3863         self.needsGit = False
3864         self.cloneBare = False
3865
3866     def defaultDestination(self, args):
3867         ## TODO: use common prefix of args?
3868         depotPath = args[0]
3869         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3870         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3871         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3872         depotDir = re.sub(r"/$", "", depotDir)
3873         return os.path.split(depotDir)[1]
3874
3875     def run(self, args):
3876         if len(args) < 1:
3877             return False
3878
3879         if self.keepRepoPath and not self.cloneDestination:
3880             sys.stderr.write("Must specify destination for --keep-path\n")
3881             sys.exit(1)
3882
3883         depotPaths = args
3884
3885         if not self.cloneDestination and len(depotPaths) > 1:
3886             self.cloneDestination = depotPaths[-1]
3887             depotPaths = depotPaths[:-1]
3888
3889         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3890         for p in depotPaths:
3891             if not p.startswith("//"):
3892                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3893                 return False
3894
3895         if not self.cloneDestination:
3896             self.cloneDestination = self.defaultDestination(args)
3897
3898         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3899
3900         if not os.path.exists(self.cloneDestination):
3901             os.makedirs(self.cloneDestination)
3902         chdir(self.cloneDestination)
3903
3904         init_cmd = [ "git", "init" ]
3905         if self.cloneBare:
3906             init_cmd.append("--bare")
3907         retcode = subprocess.call(init_cmd)
3908         if retcode:
3909             raise CalledProcessError(retcode, init_cmd)
3910
3911         if not P4Sync.run(self, depotPaths):
3912             return False
3913
3914         # create a master branch and check out a work tree
3915         if gitBranchExists(self.branch):
3916             system([ "git", "branch", "master", self.branch ])
3917             if not self.cloneBare:
3918                 system([ "git", "checkout", "-f" ])
3919         else:
3920             print('Not checking out any branch, use ' \
3921                   '"git checkout -q -b master <branch>"')
3922
3923         # auto-set this variable if invoked with --use-client-spec
3924         if self.useClientSpec_from_options:
3925             system("git config --bool git-p4.useclientspec true")
3926
3927         return True
3928
3929 class P4Unshelve(Command):
3930     def __init__(self):
3931         Command.__init__(self)
3932         self.options = []
3933         self.origin = "HEAD"
3934         self.description = "Unshelve a P4 changelist into a git commit"
3935         self.usage = "usage: %prog [options] changelist"
3936         self.options += [
3937                 optparse.make_option("--origin", dest="origin",
3938                     help="Use this base revision instead of the default (%s)" % self.origin),
3939         ]
3940         self.verbose = False
3941         self.noCommit = False
3942         self.destbranch = "refs/remotes/p4/unshelved"
3943
3944     def renameBranch(self, branch_name):
3945         """ Rename the existing branch to branch_name.N
3946         """
3947
3948         found = True
3949         for i in range(0,1000):
3950             backup_branch_name = "{0}.{1}".format(branch_name, i)
3951             if not gitBranchExists(backup_branch_name):
3952                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3953                 gitDeleteRef(branch_name)
3954                 found = True
3955                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3956                 break
3957
3958         if not found:
3959             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3960
3961     def findLastP4Revision(self, starting_point):
3962         """ Look back from starting_point for the first commit created by git-p4
3963             to find the P4 commit we are based on, and the depot-paths.
3964         """
3965
3966         for parent in (range(65535)):
3967             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3968             settings = extractSettingsGitLog(log)
3969             if 'change' in settings:
3970                 return settings
3971
3972         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3973
3974     def run(self, args):
3975         if len(args) != 1:
3976             return False
3977
3978         if not gitBranchExists(self.origin):
3979             sys.exit("origin branch {0} does not exist".format(self.origin))
3980
3981         sync = P4Sync()
3982         changes = args
3983         sync.initialParent = self.origin
3984
3985         # use the first change in the list to construct the branch to unshelve into
3986         change = changes[0]
3987
3988         # if the target branch already exists, rename it
3989         branch_name = "{0}/{1}".format(self.destbranch, change)
3990         if gitBranchExists(branch_name):
3991             self.renameBranch(branch_name)
3992         sync.branch = branch_name
3993
3994         sync.verbose = self.verbose
3995         sync.suppress_meta_comment = True
3996
3997         settings = self.findLastP4Revision(self.origin)
3998         origin_revision = settings['change']
3999         sync.depotPaths = settings['depot-paths']
4000         sync.branchPrefixes = sync.depotPaths
4001
4002         sync.openStreams()
4003         sync.loadUserMapFromCache()
4004         sync.silent = True
4005         sync.importChanges(changes, shelved=True, origin_revision=origin_revision)
4006         sync.closeStreams()
4007
4008         print("unshelved changelist {0} into {1}".format(change, branch_name))
4009
4010         return True
4011
4012 class P4Branches(Command):
4013     def __init__(self):
4014         Command.__init__(self)
4015         self.options = [ ]
4016         self.description = ("Shows the git branches that hold imports and their "
4017                             + "corresponding perforce depot paths")
4018         self.verbose = False
4019
4020     def run(self, args):
4021         if originP4BranchesExist():
4022             createOrUpdateBranchesFromOrigin()
4023
4024         cmdline = "git rev-parse --symbolic "
4025         cmdline += " --remotes"
4026
4027         for line in read_pipe_lines(cmdline):
4028             line = line.strip()
4029
4030             if not line.startswith('p4/') or line == "p4/HEAD":
4031                 continue
4032             branch = line
4033
4034             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4035             settings = extractSettingsGitLog(log)
4036
4037             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4038         return True
4039
4040 class HelpFormatter(optparse.IndentedHelpFormatter):
4041     def __init__(self):
4042         optparse.IndentedHelpFormatter.__init__(self)
4043
4044     def format_description(self, description):
4045         if description:
4046             return description + "\n"
4047         else:
4048             return ""
4049
4050 def printUsage(commands):
4051     print("usage: %s <command> [options]" % sys.argv[0])
4052     print("")
4053     print("valid commands: %s" % ", ".join(commands))
4054     print("")
4055     print("Try %s <command> --help for command specific help." % sys.argv[0])
4056     print("")
4057
4058 commands = {
4059     "debug" : P4Debug,
4060     "submit" : P4Submit,
4061     "commit" : P4Submit,
4062     "sync" : P4Sync,
4063     "rebase" : P4Rebase,
4064     "clone" : P4Clone,
4065     "rollback" : P4RollBack,
4066     "branches" : P4Branches,
4067     "unshelve" : P4Unshelve,
4068 }
4069
4070
4071 def main():
4072     if len(sys.argv[1:]) == 0:
4073         printUsage(commands.keys())
4074         sys.exit(2)
4075
4076     cmdName = sys.argv[1]
4077     try:
4078         klass = commands[cmdName]
4079         cmd = klass()
4080     except KeyError:
4081         print("unknown command %s" % cmdName)
4082         print("")
4083         printUsage(commands.keys())
4084         sys.exit(2)
4085
4086     options = cmd.options
4087     cmd.gitdir = os.environ.get("GIT_DIR", None)
4088
4089     args = sys.argv[2:]
4090
4091     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4092     if cmd.needsGit:
4093         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4094
4095     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4096                                    options,
4097                                    description = cmd.description,
4098                                    formatter = HelpFormatter())
4099
4100     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4101     global verbose
4102     verbose = cmd.verbose
4103     if cmd.needsGit:
4104         if cmd.gitdir == None:
4105             cmd.gitdir = os.path.abspath(".git")
4106             if not isValidGitDir(cmd.gitdir):
4107                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4108                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4109                 if os.path.exists(cmd.gitdir):
4110                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4111                     if len(cdup) > 0:
4112                         chdir(cdup);
4113
4114         if not isValidGitDir(cmd.gitdir):
4115             if isValidGitDir(cmd.gitdir + "/.git"):
4116                 cmd.gitdir += "/.git"
4117             else:
4118                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4119
4120         # so git commands invoked from the P4 workspace will succeed
4121         os.environ["GIT_DIR"] = cmd.gitdir
4122
4123     if not cmd.run(args):
4124         parser.print_help()
4125         sys.exit(2)
4126
4127
4128 if __name__ == '__main__':
4129     main()