merge-recursive: copy $GITHEAD strings
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 # support basestring in python3
31 try:
32     unicode = unicode
33 except NameError:
34     # 'unicode' is undefined, must be Python 3
35     str = str
36     unicode = str
37     bytes = bytes
38     basestring = (str,bytes)
39 else:
40     # 'unicode' exists, must be Python 2
41     str = str
42     unicode = unicode
43     bytes = str
44     basestring = basestring
45
46 try:
47     from subprocess import CalledProcessError
48 except ImportError:
49     # from python2.7:subprocess.py
50     # Exception classes used by this module.
51     class CalledProcessError(Exception):
52         """This exception is raised when a process run by check_call() returns
53         a non-zero exit status.  The exit status will be stored in the
54         returncode attribute."""
55         def __init__(self, returncode, cmd):
56             self.returncode = returncode
57             self.cmd = cmd
58         def __str__(self):
59             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
61 verbose = False
62
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
65
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
68
69 p4_access_checked = False
70
71 def p4_build_cmd(cmd):
72     """Build a suitable p4 command line.
73
74     This consolidates building and returning a p4 command line into one
75     location. It means that hooking into the environment, or other configuration
76     can be done more easily.
77     """
78     real_cmd = ["p4"]
79
80     user = gitConfig("git-p4.user")
81     if len(user) > 0:
82         real_cmd += ["-u",user]
83
84     password = gitConfig("git-p4.password")
85     if len(password) > 0:
86         real_cmd += ["-P", password]
87
88     port = gitConfig("git-p4.port")
89     if len(port) > 0:
90         real_cmd += ["-p", port]
91
92     host = gitConfig("git-p4.host")
93     if len(host) > 0:
94         real_cmd += ["-H", host]
95
96     client = gitConfig("git-p4.client")
97     if len(client) > 0:
98         real_cmd += ["-c", client]
99
100     retries = gitConfigInt("git-p4.retries")
101     if retries is None:
102         # Perform 3 retries by default
103         retries = 3
104     if retries > 0:
105         # Provide a way to not pass this option by setting git-p4.retries to 0
106         real_cmd += ["-r", str(retries)]
107
108     if isinstance(cmd,basestring):
109         real_cmd = ' '.join(real_cmd) + ' ' + cmd
110     else:
111         real_cmd += cmd
112
113     # now check that we can actually talk to the server
114     global p4_access_checked
115     if not p4_access_checked:
116         p4_access_checked = True    # suppress access checks in p4_check_access itself
117         p4_check_access()
118
119     return real_cmd
120
121 def git_dir(path):
122     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123         This won't automatically add ".git" to a directory.
124     """
125     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126     if not d or len(d) == 0:
127         return None
128     else:
129         return d
130
131 def chdir(path, is_client_path=False):
132     """Do chdir to the given path, and set the PWD environment
133        variable for use by P4.  It does not look at getcwd() output.
134        Since we're not using the shell, it is necessary to set the
135        PWD environment variable explicitly.
136
137        Normally, expand the path to force it to be absolute.  This
138        addresses the use of relative path names inside P4 settings,
139        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
140        as given; it looks for .p4config using PWD.
141
142        If is_client_path, the path was handed to us directly by p4,
143        and may be a symbolic link.  Do not call os.getcwd() in this
144        case, because it will cause p4 to think that PWD is not inside
145        the client path.
146        """
147
148     os.chdir(path)
149     if not is_client_path:
150         path = os.getcwd()
151     os.environ['PWD'] = path
152
153 def calcDiskFree():
154     """Return free space in bytes on the disk of the given dirname."""
155     if platform.system() == 'Windows':
156         free_bytes = ctypes.c_ulonglong(0)
157         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158         return free_bytes.value
159     else:
160         st = os.statvfs(os.getcwd())
161         return st.f_bavail * st.f_frsize
162
163 def die(msg):
164     if verbose:
165         raise Exception(msg)
166     else:
167         sys.stderr.write(msg + "\n")
168         sys.exit(1)
169
170 def write_pipe(c, stdin):
171     if verbose:
172         sys.stderr.write('Writing pipe: %s\n' % str(c))
173
174     expand = isinstance(c,basestring)
175     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176     pipe = p.stdin
177     val = pipe.write(stdin)
178     pipe.close()
179     if p.wait():
180         die('Command failed: %s' % str(c))
181
182     return val
183
184 def p4_write_pipe(c, stdin):
185     real_cmd = p4_build_cmd(c)
186     return write_pipe(real_cmd, stdin)
187
188 def read_pipe_full(c):
189     """ Read output from  command. Returns a tuple
190         of the return status, stdout text and stderr
191         text.
192     """
193     if verbose:
194         sys.stderr.write('Reading pipe: %s\n' % str(c))
195
196     expand = isinstance(c,basestring)
197     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198     (out, err) = p.communicate()
199     return (p.returncode, out, err)
200
201 def read_pipe(c, ignore_error=False):
202     """ Read output from  command. Returns the output text on
203         success. On failure, terminates execution, unless
204         ignore_error is True, when it returns an empty string.
205     """
206     (retcode, out, err) = read_pipe_full(c)
207     if retcode != 0:
208         if ignore_error:
209             out = ""
210         else:
211             die('Command failed: %s\nError: %s' % (str(c), err))
212     return out
213
214 def read_pipe_text(c):
215     """ Read output from a command with trailing whitespace stripped.
216         On error, returns None.
217     """
218     (retcode, out, err) = read_pipe_full(c)
219     if retcode != 0:
220         return None
221     else:
222         return out.rstrip()
223
224 def p4_read_pipe(c, ignore_error=False):
225     real_cmd = p4_build_cmd(c)
226     return read_pipe(real_cmd, ignore_error)
227
228 def read_pipe_lines(c):
229     if verbose:
230         sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232     expand = isinstance(c, basestring)
233     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234     pipe = p.stdout
235     val = pipe.readlines()
236     if pipe.close() or p.wait():
237         die('Command failed: %s' % str(c))
238
239     return val
240
241 def p4_read_pipe_lines(c):
242     """Specifically invoke p4 on the command supplied. """
243     real_cmd = p4_build_cmd(c)
244     return read_pipe_lines(real_cmd)
245
246 def p4_has_command(cmd):
247     """Ask p4 for help on this command.  If it returns an error, the
248        command does not exist in this version of p4."""
249     real_cmd = p4_build_cmd(["help", cmd])
250     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251                                    stderr=subprocess.PIPE)
252     p.communicate()
253     return p.returncode == 0
254
255 def p4_has_move_command():
256     """See if the move command exists, that it supports -k, and that
257        it has not been administratively disabled.  The arguments
258        must be correct, but the filenames do not have to exist.  Use
259        ones with wildcards so even if they exist, it will fail."""
260
261     if not p4_has_command("move"):
262         return False
263     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265     (out, err) = p.communicate()
266     # return code will be 1 in either case
267     if err.find("Invalid option") >= 0:
268         return False
269     if err.find("disabled") >= 0:
270         return False
271     # assume it failed because @... was invalid changelist
272     return True
273
274 def system(cmd, ignore_error=False):
275     expand = isinstance(cmd,basestring)
276     if verbose:
277         sys.stderr.write("executing %s\n" % str(cmd))
278     retcode = subprocess.call(cmd, shell=expand)
279     if retcode and not ignore_error:
280         raise CalledProcessError(retcode, cmd)
281
282     return retcode
283
284 def p4_system(cmd):
285     """Specifically invoke p4 as the system command. """
286     real_cmd = p4_build_cmd(cmd)
287     expand = isinstance(real_cmd, basestring)
288     retcode = subprocess.call(real_cmd, shell=expand)
289     if retcode:
290         raise CalledProcessError(retcode, real_cmd)
291
292 def die_bad_access(s):
293     die("failure accessing depot: {0}".format(s.rstrip()))
294
295 def p4_check_access(min_expiration=1):
296     """ Check if we can access Perforce - account still logged in
297     """
298     results = p4CmdList(["login", "-s"])
299
300     if len(results) == 0:
301         # should never get here: always get either some results, or a p4ExitCode
302         assert("could not parse response from perforce")
303
304     result = results[0]
305
306     if 'p4ExitCode' in result:
307         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308         die_bad_access("could not run p4")
309
310     code = result.get("code")
311     if not code:
312         # we get here if we couldn't connect and there was nothing to unmarshal
313         die_bad_access("could not connect")
314
315     elif code == "stat":
316         expiry = result.get("TicketExpiration")
317         if expiry:
318             expiry = int(expiry)
319             if expiry > min_expiration:
320                 # ok to carry on
321                 return
322             else:
323                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325         else:
326             # account without a timeout - all ok
327             return
328
329     elif code == "error":
330         data = result.get("data")
331         if data:
332             die_bad_access("p4 error: {0}".format(data))
333         else:
334             die_bad_access("unknown error")
335     else:
336         die_bad_access("unknown error code {0}".format(code))
337
338 _p4_version_string = None
339 def p4_version_string():
340     """Read the version string, showing just the last line, which
341        hopefully is the interesting version bit.
342
343        $ p4 -V
344        Perforce - The Fast Software Configuration Management System.
345        Copyright 1995-2011 Perforce Software.  All rights reserved.
346        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
347     """
348     global _p4_version_string
349     if not _p4_version_string:
350         a = p4_read_pipe_lines(["-V"])
351         _p4_version_string = a[-1].rstrip()
352     return _p4_version_string
353
354 def p4_integrate(src, dest):
355     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
356
357 def p4_sync(f, *options):
358     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
359
360 def p4_add(f):
361     # forcibly add file names with wildcards
362     if wildcard_present(f):
363         p4_system(["add", "-f", f])
364     else:
365         p4_system(["add", f])
366
367 def p4_delete(f):
368     p4_system(["delete", wildcard_encode(f)])
369
370 def p4_edit(f, *options):
371     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
372
373 def p4_revert(f):
374     p4_system(["revert", wildcard_encode(f)])
375
376 def p4_reopen(type, f):
377     p4_system(["reopen", "-t", type, wildcard_encode(f)])
378
379 def p4_reopen_in_change(changelist, files):
380     cmd = ["reopen", "-c", str(changelist)] + files
381     p4_system(cmd)
382
383 def p4_move(src, dest):
384     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
385
386 def p4_last_change():
387     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
388     return int(results[0]['change'])
389
390 def p4_describe(change, shelved=False):
391     """Make sure it returns a valid result by checking for
392        the presence of field "time".  Return a dict of the
393        results."""
394
395     cmd = ["describe", "-s"]
396     if shelved:
397         cmd += ["-S"]
398     cmd += [str(change)]
399
400     ds = p4CmdList(cmd, skip_info=True)
401     if len(ds) != 1:
402         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
403
404     d = ds[0]
405
406     if "p4ExitCode" in d:
407         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
408                                                       str(d)))
409     if "code" in d:
410         if d["code"] == "error":
411             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
412
413     if "time" not in d:
414         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
415
416     return d
417
418 #
419 # Canonicalize the p4 type and return a tuple of the
420 # base type, plus any modifiers.  See "p4 help filetypes"
421 # for a list and explanation.
422 #
423 def split_p4_type(p4type):
424
425     p4_filetypes_historical = {
426         "ctempobj": "binary+Sw",
427         "ctext": "text+C",
428         "cxtext": "text+Cx",
429         "ktext": "text+k",
430         "kxtext": "text+kx",
431         "ltext": "text+F",
432         "tempobj": "binary+FSw",
433         "ubinary": "binary+F",
434         "uresource": "resource+F",
435         "uxbinary": "binary+Fx",
436         "xbinary": "binary+x",
437         "xltext": "text+Fx",
438         "xtempobj": "binary+Swx",
439         "xtext": "text+x",
440         "xunicode": "unicode+x",
441         "xutf16": "utf16+x",
442     }
443     if p4type in p4_filetypes_historical:
444         p4type = p4_filetypes_historical[p4type]
445     mods = ""
446     s = p4type.split("+")
447     base = s[0]
448     mods = ""
449     if len(s) > 1:
450         mods = s[1]
451     return (base, mods)
452
453 #
454 # return the raw p4 type of a file (text, text+ko, etc)
455 #
456 def p4_type(f):
457     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
458     return results[0]['headType']
459
460 #
461 # Given a type base and modifier, return a regexp matching
462 # the keywords that can be expanded in the file
463 #
464 def p4_keywords_regexp_for_type(base, type_mods):
465     if base in ("text", "unicode", "binary"):
466         kwords = None
467         if "ko" in type_mods:
468             kwords = 'Id|Header'
469         elif "k" in type_mods:
470             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
471         else:
472             return None
473         pattern = r"""
474             \$              # Starts with a dollar, followed by...
475             (%s)            # one of the keywords, followed by...
476             (:[^$\n]+)?     # possibly an old expansion, followed by...
477             \$              # another dollar
478             """ % kwords
479         return pattern
480     else:
481         return None
482
483 #
484 # Given a file, return a regexp matching the possible
485 # RCS keywords that will be expanded, or None for files
486 # with kw expansion turned off.
487 #
488 def p4_keywords_regexp_for_file(file):
489     if not os.path.exists(file):
490         return None
491     else:
492         (type_base, type_mods) = split_p4_type(p4_type(file))
493         return p4_keywords_regexp_for_type(type_base, type_mods)
494
495 def setP4ExecBit(file, mode):
496     # Reopens an already open file and changes the execute bit to match
497     # the execute bit setting in the passed in mode.
498
499     p4Type = "+x"
500
501     if not isModeExec(mode):
502         p4Type = getP4OpenedType(file)
503         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
504         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
505         if p4Type[-1] == "+":
506             p4Type = p4Type[0:-1]
507
508     p4_reopen(p4Type, file)
509
510 def getP4OpenedType(file):
511     # Returns the perforce file type for the given file.
512
513     result = p4_read_pipe(["opened", wildcard_encode(file)])
514     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
515     if match:
516         return match.group(1)
517     else:
518         die("Could not determine file type for %s (result: '%s')" % (file, result))
519
520 # Return the set of all p4 labels
521 def getP4Labels(depotPaths):
522     labels = set()
523     if isinstance(depotPaths,basestring):
524         depotPaths = [depotPaths]
525
526     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
527         label = l['label']
528         labels.add(label)
529
530     return labels
531
532 # Return the set of all git tags
533 def getGitTags():
534     gitTags = set()
535     for line in read_pipe_lines(["git", "tag"]):
536         tag = line.strip()
537         gitTags.add(tag)
538     return gitTags
539
540 def diffTreePattern():
541     # This is a simple generator for the diff tree regex pattern. This could be
542     # a class variable if this and parseDiffTreeEntry were a part of a class.
543     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
544     while True:
545         yield pattern
546
547 def parseDiffTreeEntry(entry):
548     """Parses a single diff tree entry into its component elements.
549
550     See git-diff-tree(1) manpage for details about the format of the diff
551     output. This method returns a dictionary with the following elements:
552
553     src_mode - The mode of the source file
554     dst_mode - The mode of the destination file
555     src_sha1 - The sha1 for the source file
556     dst_sha1 - The sha1 fr the destination file
557     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
558     status_score - The score for the status (applicable for 'C' and 'R'
559                    statuses). This is None if there is no score.
560     src - The path for the source file.
561     dst - The path for the destination file. This is only present for
562           copy or renames. If it is not present, this is None.
563
564     If the pattern is not matched, None is returned."""
565
566     match = diffTreePattern().next().match(entry)
567     if match:
568         return {
569             'src_mode': match.group(1),
570             'dst_mode': match.group(2),
571             'src_sha1': match.group(3),
572             'dst_sha1': match.group(4),
573             'status': match.group(5),
574             'status_score': match.group(6),
575             'src': match.group(7),
576             'dst': match.group(10)
577         }
578     return None
579
580 def isModeExec(mode):
581     # Returns True if the given git mode represents an executable file,
582     # otherwise False.
583     return mode[-3:] == "755"
584
585 class P4Exception(Exception):
586     """ Base class for exceptions from the p4 client """
587     def __init__(self, exit_code):
588         self.p4ExitCode = exit_code
589
590 class P4ServerException(P4Exception):
591     """ Base class for exceptions where we get some kind of marshalled up result from the server """
592     def __init__(self, exit_code, p4_result):
593         super(P4ServerException, self).__init__(exit_code)
594         self.p4_result = p4_result
595         self.code = p4_result[0]['code']
596         self.data = p4_result[0]['data']
597
598 class P4RequestSizeException(P4ServerException):
599     """ One of the maxresults or maxscanrows errors """
600     def __init__(self, exit_code, p4_result, limit):
601         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
602         self.limit = limit
603
604 def isModeExecChanged(src_mode, dst_mode):
605     return isModeExec(src_mode) != isModeExec(dst_mode)
606
607 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
608         errors_as_exceptions=False):
609
610     if isinstance(cmd,basestring):
611         cmd = "-G " + cmd
612         expand = True
613     else:
614         cmd = ["-G"] + cmd
615         expand = False
616
617     cmd = p4_build_cmd(cmd)
618     if verbose:
619         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
620
621     # Use a temporary file to avoid deadlocks without
622     # subprocess.communicate(), which would put another copy
623     # of stdout into memory.
624     stdin_file = None
625     if stdin is not None:
626         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
627         if isinstance(stdin,basestring):
628             stdin_file.write(stdin)
629         else:
630             for i in stdin:
631                 stdin_file.write(i + '\n')
632         stdin_file.flush()
633         stdin_file.seek(0)
634
635     p4 = subprocess.Popen(cmd,
636                           shell=expand,
637                           stdin=stdin_file,
638                           stdout=subprocess.PIPE)
639
640     result = []
641     try:
642         while True:
643             entry = marshal.load(p4.stdout)
644             if skip_info:
645                 if 'code' in entry and entry['code'] == 'info':
646                     continue
647             if cb is not None:
648                 cb(entry)
649             else:
650                 result.append(entry)
651     except EOFError:
652         pass
653     exitCode = p4.wait()
654     if exitCode != 0:
655         if errors_as_exceptions:
656             if len(result) > 0:
657                 data = result[0].get('data')
658                 if data:
659                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
660                     if not m:
661                         m = re.search('Request too large \(over (\d+)\)', data)
662
663                     if m:
664                         limit = int(m.group(1))
665                         raise P4RequestSizeException(exitCode, result, limit)
666
667                 raise P4ServerException(exitCode, result)
668             else:
669                 raise P4Exception(exitCode)
670         else:
671             entry = {}
672             entry["p4ExitCode"] = exitCode
673             result.append(entry)
674
675     return result
676
677 def p4Cmd(cmd):
678     list = p4CmdList(cmd)
679     result = {}
680     for entry in list:
681         result.update(entry)
682     return result;
683
684 def p4Where(depotPath):
685     if not depotPath.endswith("/"):
686         depotPath += "/"
687     depotPathLong = depotPath + "..."
688     outputList = p4CmdList(["where", depotPathLong])
689     output = None
690     for entry in outputList:
691         if "depotFile" in entry:
692             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
693             # The base path always ends with "/...".
694             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
695                 output = entry
696                 break
697         elif "data" in entry:
698             data = entry.get("data")
699             space = data.find(" ")
700             if data[:space] == depotPath:
701                 output = entry
702                 break
703     if output == None:
704         return ""
705     if output["code"] == "error":
706         return ""
707     clientPath = ""
708     if "path" in output:
709         clientPath = output.get("path")
710     elif "data" in output:
711         data = output.get("data")
712         lastSpace = data.rfind(" ")
713         clientPath = data[lastSpace + 1:]
714
715     if clientPath.endswith("..."):
716         clientPath = clientPath[:-3]
717     return clientPath
718
719 def currentGitBranch():
720     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
721
722 def isValidGitDir(path):
723     return git_dir(path) != None
724
725 def parseRevision(ref):
726     return read_pipe("git rev-parse %s" % ref).strip()
727
728 def branchExists(ref):
729     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
730                      ignore_error=True)
731     return len(rev) > 0
732
733 def extractLogMessageFromGitCommit(commit):
734     logMessage = ""
735
736     ## fixme: title is first line of commit, not 1st paragraph.
737     foundTitle = False
738     for log in read_pipe_lines("git cat-file commit %s" % commit):
739        if not foundTitle:
740            if len(log) == 1:
741                foundTitle = True
742            continue
743
744        logMessage += log
745     return logMessage
746
747 def extractSettingsGitLog(log):
748     values = {}
749     for line in log.split("\n"):
750         line = line.strip()
751         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
752         if not m:
753             continue
754
755         assignments = m.group(1).split (':')
756         for a in assignments:
757             vals = a.split ('=')
758             key = vals[0].strip()
759             val = ('='.join (vals[1:])).strip()
760             if val.endswith ('\"') and val.startswith('"'):
761                 val = val[1:-1]
762
763             values[key] = val
764
765     paths = values.get("depot-paths")
766     if not paths:
767         paths = values.get("depot-path")
768     if paths:
769         values['depot-paths'] = paths.split(',')
770     return values
771
772 def gitBranchExists(branch):
773     proc = subprocess.Popen(["git", "rev-parse", branch],
774                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
775     return proc.wait() == 0;
776
777 def gitUpdateRef(ref, newvalue):
778     subprocess.check_call(["git", "update-ref", ref, newvalue])
779
780 def gitDeleteRef(ref):
781     subprocess.check_call(["git", "update-ref", "-d", ref])
782
783 _gitConfig = {}
784
785 def gitConfig(key, typeSpecifier=None):
786     if key not in _gitConfig:
787         cmd = [ "git", "config" ]
788         if typeSpecifier:
789             cmd += [ typeSpecifier ]
790         cmd += [ key ]
791         s = read_pipe(cmd, ignore_error=True)
792         _gitConfig[key] = s.strip()
793     return _gitConfig[key]
794
795 def gitConfigBool(key):
796     """Return a bool, using git config --bool.  It is True only if the
797        variable is set to true, and False if set to false or not present
798        in the config."""
799
800     if key not in _gitConfig:
801         _gitConfig[key] = gitConfig(key, '--bool') == "true"
802     return _gitConfig[key]
803
804 def gitConfigInt(key):
805     if key not in _gitConfig:
806         cmd = [ "git", "config", "--int", key ]
807         s = read_pipe(cmd, ignore_error=True)
808         v = s.strip()
809         try:
810             _gitConfig[key] = int(gitConfig(key, '--int'))
811         except ValueError:
812             _gitConfig[key] = None
813     return _gitConfig[key]
814
815 def gitConfigList(key):
816     if key not in _gitConfig:
817         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
818         _gitConfig[key] = s.strip().splitlines()
819         if _gitConfig[key] == ['']:
820             _gitConfig[key] = []
821     return _gitConfig[key]
822
823 def p4BranchesInGit(branchesAreInRemotes=True):
824     """Find all the branches whose names start with "p4/", looking
825        in remotes or heads as specified by the argument.  Return
826        a dictionary of { branch: revision } for each one found.
827        The branch names are the short names, without any
828        "p4/" prefix."""
829
830     branches = {}
831
832     cmdline = "git rev-parse --symbolic "
833     if branchesAreInRemotes:
834         cmdline += "--remotes"
835     else:
836         cmdline += "--branches"
837
838     for line in read_pipe_lines(cmdline):
839         line = line.strip()
840
841         # only import to p4/
842         if not line.startswith('p4/'):
843             continue
844         # special symbolic ref to p4/master
845         if line == "p4/HEAD":
846             continue
847
848         # strip off p4/ prefix
849         branch = line[len("p4/"):]
850
851         branches[branch] = parseRevision(line)
852
853     return branches
854
855 def branch_exists(branch):
856     """Make sure that the given ref name really exists."""
857
858     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
859     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
860     out, _ = p.communicate()
861     if p.returncode:
862         return False
863     # expect exactly one line of output: the branch name
864     return out.rstrip() == branch
865
866 def findUpstreamBranchPoint(head = "HEAD"):
867     branches = p4BranchesInGit()
868     # map from depot-path to branch name
869     branchByDepotPath = {}
870     for branch in branches.keys():
871         tip = branches[branch]
872         log = extractLogMessageFromGitCommit(tip)
873         settings = extractSettingsGitLog(log)
874         if "depot-paths" in settings:
875             paths = ",".join(settings["depot-paths"])
876             branchByDepotPath[paths] = "remotes/p4/" + branch
877
878     settings = None
879     parent = 0
880     while parent < 65535:
881         commit = head + "~%s" % parent
882         log = extractLogMessageFromGitCommit(commit)
883         settings = extractSettingsGitLog(log)
884         if "depot-paths" in settings:
885             paths = ",".join(settings["depot-paths"])
886             if paths in branchByDepotPath:
887                 return [branchByDepotPath[paths], settings]
888
889         parent = parent + 1
890
891     return ["", settings]
892
893 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
894     if not silent:
895         print("Creating/updating branch(es) in %s based on origin branch(es)"
896                % localRefPrefix)
897
898     originPrefix = "origin/p4/"
899
900     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
901         line = line.strip()
902         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
903             continue
904
905         headName = line[len(originPrefix):]
906         remoteHead = localRefPrefix + headName
907         originHead = line
908
909         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
910         if ('depot-paths' not in original
911             or 'change' not in original):
912             continue
913
914         update = False
915         if not gitBranchExists(remoteHead):
916             if verbose:
917                 print("creating %s" % remoteHead)
918             update = True
919         else:
920             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
921             if 'change' in settings:
922                 if settings['depot-paths'] == original['depot-paths']:
923                     originP4Change = int(original['change'])
924                     p4Change = int(settings['change'])
925                     if originP4Change > p4Change:
926                         print("%s (%s) is newer than %s (%s). "
927                                "Updating p4 branch from origin."
928                                % (originHead, originP4Change,
929                                   remoteHead, p4Change))
930                         update = True
931                 else:
932                     print("Ignoring: %s was imported from %s while "
933                            "%s was imported from %s"
934                            % (originHead, ','.join(original['depot-paths']),
935                               remoteHead, ','.join(settings['depot-paths'])))
936
937         if update:
938             system("git update-ref %s %s" % (remoteHead, originHead))
939
940 def originP4BranchesExist():
941         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
942
943
944 def p4ParseNumericChangeRange(parts):
945     changeStart = int(parts[0][1:])
946     if parts[1] == '#head':
947         changeEnd = p4_last_change()
948     else:
949         changeEnd = int(parts[1])
950
951     return (changeStart, changeEnd)
952
953 def chooseBlockSize(blockSize):
954     if blockSize:
955         return blockSize
956     else:
957         return defaultBlockSize
958
959 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
960     assert depotPaths
961
962     # Parse the change range into start and end. Try to find integer
963     # revision ranges as these can be broken up into blocks to avoid
964     # hitting server-side limits (maxrows, maxscanresults). But if
965     # that doesn't work, fall back to using the raw revision specifier
966     # strings, without using block mode.
967
968     if changeRange is None or changeRange == '':
969         changeStart = 1
970         changeEnd = p4_last_change()
971         block_size = chooseBlockSize(requestedBlockSize)
972     else:
973         parts = changeRange.split(',')
974         assert len(parts) == 2
975         try:
976             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
977             block_size = chooseBlockSize(requestedBlockSize)
978         except ValueError:
979             changeStart = parts[0][1:]
980             changeEnd = parts[1]
981             if requestedBlockSize:
982                 die("cannot use --changes-block-size with non-numeric revisions")
983             block_size = None
984
985     changes = set()
986
987     # Retrieve changes a block at a time, to prevent running
988     # into a MaxResults/MaxScanRows error from the server. If
989     # we _do_ hit one of those errors, turn down the block size
990
991     while True:
992         cmd = ['changes']
993
994         if block_size:
995             end = min(changeEnd, changeStart + block_size)
996             revisionRange = "%d,%d" % (changeStart, end)
997         else:
998             revisionRange = "%s,%s" % (changeStart, changeEnd)
999
1000         for p in depotPaths:
1001             cmd += ["%s...@%s" % (p, revisionRange)]
1002
1003         # fetch the changes
1004         try:
1005             result = p4CmdList(cmd, errors_as_exceptions=True)
1006         except P4RequestSizeException as e:
1007             if not block_size:
1008                 block_size = e.limit
1009             elif block_size > e.limit:
1010                 block_size = e.limit
1011             else:
1012                 block_size = max(2, block_size // 2)
1013
1014             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1015             continue
1016         except P4Exception as e:
1017             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1018
1019         # Insert changes in chronological order
1020         for entry in reversed(result):
1021             if 'change' not in entry:
1022                 continue
1023             changes.add(int(entry['change']))
1024
1025         if not block_size:
1026             break
1027
1028         if end >= changeEnd:
1029             break
1030
1031         changeStart = end + 1
1032
1033     changes = sorted(changes)
1034     return changes
1035
1036 def p4PathStartsWith(path, prefix):
1037     # This method tries to remedy a potential mixed-case issue:
1038     #
1039     # If UserA adds  //depot/DirA/file1
1040     # and UserB adds //depot/dira/file2
1041     #
1042     # we may or may not have a problem. If you have core.ignorecase=true,
1043     # we treat DirA and dira as the same directory
1044     if gitConfigBool("core.ignorecase"):
1045         return path.lower().startswith(prefix.lower())
1046     return path.startswith(prefix)
1047
1048 def getClientSpec():
1049     """Look at the p4 client spec, create a View() object that contains
1050        all the mappings, and return it."""
1051
1052     specList = p4CmdList("client -o")
1053     if len(specList) != 1:
1054         die('Output from "client -o" is %d lines, expecting 1' %
1055             len(specList))
1056
1057     # dictionary of all client parameters
1058     entry = specList[0]
1059
1060     # the //client/ name
1061     client_name = entry["Client"]
1062
1063     # just the keys that start with "View"
1064     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1065
1066     # hold this new View
1067     view = View(client_name)
1068
1069     # append the lines, in order, to the view
1070     for view_num in range(len(view_keys)):
1071         k = "View%d" % view_num
1072         if k not in view_keys:
1073             die("Expected view key %s missing" % k)
1074         view.append(entry[k])
1075
1076     return view
1077
1078 def getClientRoot():
1079     """Grab the client directory."""
1080
1081     output = p4CmdList("client -o")
1082     if len(output) != 1:
1083         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1084
1085     entry = output[0]
1086     if "Root" not in entry:
1087         die('Client has no "Root"')
1088
1089     return entry["Root"]
1090
1091 #
1092 # P4 wildcards are not allowed in filenames.  P4 complains
1093 # if you simply add them, but you can force it with "-f", in
1094 # which case it translates them into %xx encoding internally.
1095 #
1096 def wildcard_decode(path):
1097     # Search for and fix just these four characters.  Do % last so
1098     # that fixing it does not inadvertently create new %-escapes.
1099     # Cannot have * in a filename in windows; untested as to
1100     # what p4 would do in such a case.
1101     if not platform.system() == "Windows":
1102         path = path.replace("%2A", "*")
1103     path = path.replace("%23", "#") \
1104                .replace("%40", "@") \
1105                .replace("%25", "%")
1106     return path
1107
1108 def wildcard_encode(path):
1109     # do % first to avoid double-encoding the %s introduced here
1110     path = path.replace("%", "%25") \
1111                .replace("*", "%2A") \
1112                .replace("#", "%23") \
1113                .replace("@", "%40")
1114     return path
1115
1116 def wildcard_present(path):
1117     m = re.search("[*#@%]", path)
1118     return m is not None
1119
1120 class LargeFileSystem(object):
1121     """Base class for large file system support."""
1122
1123     def __init__(self, writeToGitStream):
1124         self.largeFiles = set()
1125         self.writeToGitStream = writeToGitStream
1126
1127     def generatePointer(self, cloneDestination, contentFile):
1128         """Return the content of a pointer file that is stored in Git instead of
1129            the actual content."""
1130         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1131
1132     def pushFile(self, localLargeFile):
1133         """Push the actual content which is not stored in the Git repository to
1134            a server."""
1135         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1136
1137     def hasLargeFileExtension(self, relPath):
1138         return reduce(
1139             lambda a, b: a or b,
1140             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1141             False
1142         )
1143
1144     def generateTempFile(self, contents):
1145         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146         for d in contents:
1147             contentFile.write(d)
1148         contentFile.close()
1149         return contentFile.name
1150
1151     def exceedsLargeFileThreshold(self, relPath, contents):
1152         if gitConfigInt('git-p4.largeFileThreshold'):
1153             contentsSize = sum(len(d) for d in contents)
1154             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1155                 return True
1156         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1157             contentsSize = sum(len(d) for d in contents)
1158             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159                 return False
1160             contentTempFile = self.generateTempFile(contents)
1161             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1162             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1163             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1164             zf.close()
1165             compressedContentsSize = zf.infolist()[0].compress_size
1166             os.remove(contentTempFile)
1167             os.remove(compressedContentFile.name)
1168             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169                 return True
1170         return False
1171
1172     def addLargeFile(self, relPath):
1173         self.largeFiles.add(relPath)
1174
1175     def removeLargeFile(self, relPath):
1176         self.largeFiles.remove(relPath)
1177
1178     def isLargeFile(self, relPath):
1179         return relPath in self.largeFiles
1180
1181     def processContent(self, git_mode, relPath, contents):
1182         """Processes the content of git fast import. This method decides if a
1183            file is stored in the large file system and handles all necessary
1184            steps."""
1185         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186             contentTempFile = self.generateTempFile(contents)
1187             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188             if pointer_git_mode:
1189                 git_mode = pointer_git_mode
1190             if localLargeFile:
1191                 # Move temp file to final location in large file system
1192                 largeFileDir = os.path.dirname(localLargeFile)
1193                 if not os.path.isdir(largeFileDir):
1194                     os.makedirs(largeFileDir)
1195                 shutil.move(contentTempFile, localLargeFile)
1196                 self.addLargeFile(relPath)
1197                 if gitConfigBool('git-p4.largeFilePush'):
1198                     self.pushFile(localLargeFile)
1199                 if verbose:
1200                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201         return (git_mode, contents)
1202
1203 class MockLFS(LargeFileSystem):
1204     """Mock large file system for testing."""
1205
1206     def generatePointer(self, contentFile):
1207         """The pointer content is the original content prefixed with "pointer-".
1208            The local filename of the large file storage is derived from the file content.
1209            """
1210         with open(contentFile, 'r') as f:
1211             content = next(f)
1212             gitMode = '100644'
1213             pointerContents = 'pointer-' + content
1214             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215             return (gitMode, pointerContents, localLargeFile)
1216
1217     def pushFile(self, localLargeFile):
1218         """The remote filename of the large file storage is the same as the local
1219            one but in a different directory.
1220            """
1221         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222         if not os.path.exists(remotePath):
1223             os.makedirs(remotePath)
1224         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1225
1226 class GitLFS(LargeFileSystem):
1227     """Git LFS as backend for the git-p4 large file system.
1228        See https://git-lfs.github.com/ for details."""
1229
1230     def __init__(self, *args):
1231         LargeFileSystem.__init__(self, *args)
1232         self.baseGitAttributes = []
1233
1234     def generatePointer(self, contentFile):
1235         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236            mode and content which is stored in the Git repository instead of
1237            the actual content. Return also the new location of the actual
1238            content.
1239            """
1240         if os.path.getsize(contentFile) == 0:
1241             return (None, '', None)
1242
1243         pointerProcess = subprocess.Popen(
1244             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245             stdout=subprocess.PIPE
1246         )
1247         pointerFile = pointerProcess.stdout.read()
1248         if pointerProcess.wait():
1249             os.remove(contentFile)
1250             die('git-lfs pointer command failed. Did you install the extension?')
1251
1252         # Git LFS removed the preamble in the output of the 'pointer' command
1253         # starting from version 1.2.0. Check for the preamble here to support
1254         # earlier versions.
1255         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256         if pointerFile.startswith('Git LFS pointer for'):
1257             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1258
1259         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260         localLargeFile = os.path.join(
1261             os.getcwd(),
1262             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1263             oid,
1264         )
1265         # LFS Spec states that pointer files should not have the executable bit set.
1266         gitMode = '100644'
1267         return (gitMode, pointerFile, localLargeFile)
1268
1269     def pushFile(self, localLargeFile):
1270         uploadProcess = subprocess.Popen(
1271             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1272         )
1273         if uploadProcess.wait():
1274             die('git-lfs push command failed. Did you define a remote?')
1275
1276     def generateGitAttributes(self):
1277         return (
1278             self.baseGitAttributes +
1279             [
1280                 '\n',
1281                 '#\n',
1282                 '# Git LFS (see https://git-lfs.github.com/)\n',
1283                 '#\n',
1284             ] +
1285             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1286                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1287             ] +
1288             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1289                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1290             ]
1291         )
1292
1293     def addLargeFile(self, relPath):
1294         LargeFileSystem.addLargeFile(self, relPath)
1295         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1296
1297     def removeLargeFile(self, relPath):
1298         LargeFileSystem.removeLargeFile(self, relPath)
1299         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1300
1301     def processContent(self, git_mode, relPath, contents):
1302         if relPath == '.gitattributes':
1303             self.baseGitAttributes = contents
1304             return (git_mode, self.generateGitAttributes())
1305         else:
1306             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1307
1308 class Command:
1309     def __init__(self):
1310         self.usage = "usage: %prog [options]"
1311         self.needsGit = True
1312         self.verbose = False
1313
1314     # This is required for the "append" cloneExclude action
1315     def ensure_value(self, attr, value):
1316         if not hasattr(self, attr) or getattr(self, attr) is None:
1317             setattr(self, attr, value)
1318         return getattr(self, attr)
1319
1320 class P4UserMap:
1321     def __init__(self):
1322         self.userMapFromPerforceServer = False
1323         self.myP4UserId = None
1324
1325     def p4UserId(self):
1326         if self.myP4UserId:
1327             return self.myP4UserId
1328
1329         results = p4CmdList("user -o")
1330         for r in results:
1331             if 'User' in r:
1332                 self.myP4UserId = r['User']
1333                 return r['User']
1334         die("Could not find your p4 user id")
1335
1336     def p4UserIsMe(self, p4User):
1337         # return True if the given p4 user is actually me
1338         me = self.p4UserId()
1339         if not p4User or p4User != me:
1340             return False
1341         else:
1342             return True
1343
1344     def getUserCacheFilename(self):
1345         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1346         return home + "/.gitp4-usercache.txt"
1347
1348     def getUserMapFromPerforceServer(self):
1349         if self.userMapFromPerforceServer:
1350             return
1351         self.users = {}
1352         self.emails = {}
1353
1354         for output in p4CmdList("users"):
1355             if "User" not in output:
1356                 continue
1357             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1358             self.emails[output["Email"]] = output["User"]
1359
1360         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1361         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1362             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1363             if mapUser and len(mapUser[0]) == 3:
1364                 user = mapUser[0][0]
1365                 fullname = mapUser[0][1]
1366                 email = mapUser[0][2]
1367                 self.users[user] = fullname + " <" + email + ">"
1368                 self.emails[email] = user
1369
1370         s = ''
1371         for (key, val) in self.users.items():
1372             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1373
1374         open(self.getUserCacheFilename(), "wb").write(s)
1375         self.userMapFromPerforceServer = True
1376
1377     def loadUserMapFromCache(self):
1378         self.users = {}
1379         self.userMapFromPerforceServer = False
1380         try:
1381             cache = open(self.getUserCacheFilename(), "rb")
1382             lines = cache.readlines()
1383             cache.close()
1384             for line in lines:
1385                 entry = line.strip().split("\t")
1386                 self.users[entry[0]] = entry[1]
1387         except IOError:
1388             self.getUserMapFromPerforceServer()
1389
1390 class P4Debug(Command):
1391     def __init__(self):
1392         Command.__init__(self)
1393         self.options = []
1394         self.description = "A tool to debug the output of p4 -G."
1395         self.needsGit = False
1396
1397     def run(self, args):
1398         j = 0
1399         for output in p4CmdList(args):
1400             print('Element: %d' % j)
1401             j += 1
1402             print(output)
1403         return True
1404
1405 class P4RollBack(Command):
1406     def __init__(self):
1407         Command.__init__(self)
1408         self.options = [
1409             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1410         ]
1411         self.description = "A tool to debug the multi-branch import. Don't use :)"
1412         self.rollbackLocalBranches = False
1413
1414     def run(self, args):
1415         if len(args) != 1:
1416             return False
1417         maxChange = int(args[0])
1418
1419         if "p4ExitCode" in p4Cmd("changes -m 1"):
1420             die("Problems executing p4");
1421
1422         if self.rollbackLocalBranches:
1423             refPrefix = "refs/heads/"
1424             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1425         else:
1426             refPrefix = "refs/remotes/"
1427             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1428
1429         for line in lines:
1430             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1431                 line = line.strip()
1432                 ref = refPrefix + line
1433                 log = extractLogMessageFromGitCommit(ref)
1434                 settings = extractSettingsGitLog(log)
1435
1436                 depotPaths = settings['depot-paths']
1437                 change = settings['change']
1438
1439                 changed = False
1440
1441                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1442                                                            for p in depotPaths]))) == 0:
1443                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1444                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1445                     continue
1446
1447                 while change and int(change) > maxChange:
1448                     changed = True
1449                     if self.verbose:
1450                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1451                     system("git update-ref %s \"%s^\"" % (ref, ref))
1452                     log = extractLogMessageFromGitCommit(ref)
1453                     settings =  extractSettingsGitLog(log)
1454
1455
1456                     depotPaths = settings['depot-paths']
1457                     change = settings['change']
1458
1459                 if changed:
1460                     print("%s rewound to %s" % (ref, change))
1461
1462         return True
1463
1464 class P4Submit(Command, P4UserMap):
1465
1466     conflict_behavior_choices = ("ask", "skip", "quit")
1467
1468     def __init__(self):
1469         Command.__init__(self)
1470         P4UserMap.__init__(self)
1471         self.options = [
1472                 optparse.make_option("--origin", dest="origin"),
1473                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1474                 # preserve the user, requires relevant p4 permissions
1475                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1476                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1477                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1478                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1479                 optparse.make_option("--conflict", dest="conflict_behavior",
1480                                      choices=self.conflict_behavior_choices),
1481                 optparse.make_option("--branch", dest="branch"),
1482                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1483                                      help="Shelve instead of submit. Shelved files are reverted, "
1484                                      "restoring the workspace to the state before the shelve"),
1485                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1486                                      metavar="CHANGELIST",
1487                                      help="update an existing shelved changelist, implies --shelve, "
1488                                            "repeat in-order for multiple shelved changelists"),
1489                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1490                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1491                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1492                                      help="Disable rebase after submit is completed. Can be useful if you "
1493                                      "work from a local git branch that is not master"),
1494                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1495                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1496         ]
1497         self.description = """Submit changes from git to the perforce depot.\n
1498     The `p4-pre-submit` hook is executed if it exists and is executable.
1499     The hook takes no parameters and nothing from standard input. Exiting with
1500     non-zero status from this script prevents `git-p4 submit` from launching.
1501
1502     One usage scenario is to run unit tests in the hook."""
1503
1504         self.usage += " [name of git branch to submit into perforce depot]"
1505         self.origin = ""
1506         self.detectRenames = False
1507         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1508         self.dry_run = False
1509         self.shelve = False
1510         self.update_shelve = list()
1511         self.commit = ""
1512         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1513         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1514         self.prepare_p4_only = False
1515         self.conflict_behavior = None
1516         self.isWindows = (platform.system() == "Windows")
1517         self.exportLabels = False
1518         self.p4HasMoveCommand = p4_has_move_command()
1519         self.branch = None
1520
1521         if gitConfig('git-p4.largeFileSystem'):
1522             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1523
1524     def check(self):
1525         if len(p4CmdList("opened ...")) > 0:
1526             die("You have files opened with perforce! Close them before starting the sync.")
1527
1528     def separate_jobs_from_description(self, message):
1529         """Extract and return a possible Jobs field in the commit
1530            message.  It goes into a separate section in the p4 change
1531            specification.
1532
1533            A jobs line starts with "Jobs:" and looks like a new field
1534            in a form.  Values are white-space separated on the same
1535            line or on following lines that start with a tab.
1536
1537            This does not parse and extract the full git commit message
1538            like a p4 form.  It just sees the Jobs: line as a marker
1539            to pass everything from then on directly into the p4 form,
1540            but outside the description section.
1541
1542            Return a tuple (stripped log message, jobs string)."""
1543
1544         m = re.search(r'^Jobs:', message, re.MULTILINE)
1545         if m is None:
1546             return (message, None)
1547
1548         jobtext = message[m.start():]
1549         stripped_message = message[:m.start()].rstrip()
1550         return (stripped_message, jobtext)
1551
1552     def prepareLogMessage(self, template, message, jobs):
1553         """Edits the template returned from "p4 change -o" to insert
1554            the message in the Description field, and the jobs text in
1555            the Jobs field."""
1556         result = ""
1557
1558         inDescriptionSection = False
1559
1560         for line in template.split("\n"):
1561             if line.startswith("#"):
1562                 result += line + "\n"
1563                 continue
1564
1565             if inDescriptionSection:
1566                 if line.startswith("Files:") or line.startswith("Jobs:"):
1567                     inDescriptionSection = False
1568                     # insert Jobs section
1569                     if jobs:
1570                         result += jobs + "\n"
1571                 else:
1572                     continue
1573             else:
1574                 if line.startswith("Description:"):
1575                     inDescriptionSection = True
1576                     line += "\n"
1577                     for messageLine in message.split("\n"):
1578                         line += "\t" + messageLine + "\n"
1579
1580             result += line + "\n"
1581
1582         return result
1583
1584     def patchRCSKeywords(self, file, pattern):
1585         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1586         (handle, outFileName) = tempfile.mkstemp(dir='.')
1587         try:
1588             outFile = os.fdopen(handle, "w+")
1589             inFile = open(file, "r")
1590             regexp = re.compile(pattern, re.VERBOSE)
1591             for line in inFile.readlines():
1592                 line = regexp.sub(r'$\1$', line)
1593                 outFile.write(line)
1594             inFile.close()
1595             outFile.close()
1596             # Forcibly overwrite the original file
1597             os.unlink(file)
1598             shutil.move(outFileName, file)
1599         except:
1600             # cleanup our temporary file
1601             os.unlink(outFileName)
1602             print("Failed to strip RCS keywords in %s" % file)
1603             raise
1604
1605         print("Patched up RCS keywords in %s" % file)
1606
1607     def p4UserForCommit(self,id):
1608         # Return the tuple (perforce user,git email) for a given git commit id
1609         self.getUserMapFromPerforceServer()
1610         gitEmail = read_pipe(["git", "log", "--max-count=1",
1611                               "--format=%ae", id])
1612         gitEmail = gitEmail.strip()
1613         if gitEmail not in self.emails:
1614             return (None,gitEmail)
1615         else:
1616             return (self.emails[gitEmail],gitEmail)
1617
1618     def checkValidP4Users(self,commits):
1619         # check if any git authors cannot be mapped to p4 users
1620         for id in commits:
1621             (user,email) = self.p4UserForCommit(id)
1622             if not user:
1623                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1624                 if gitConfigBool("git-p4.allowMissingP4Users"):
1625                     print("%s" % msg)
1626                 else:
1627                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1628
1629     def lastP4Changelist(self):
1630         # Get back the last changelist number submitted in this client spec. This
1631         # then gets used to patch up the username in the change. If the same
1632         # client spec is being used by multiple processes then this might go
1633         # wrong.
1634         results = p4CmdList("client -o")        # find the current client
1635         client = None
1636         for r in results:
1637             if 'Client' in r:
1638                 client = r['Client']
1639                 break
1640         if not client:
1641             die("could not get client spec")
1642         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1643         for r in results:
1644             if 'change' in r:
1645                 return r['change']
1646         die("Could not get changelist number for last submit - cannot patch up user details")
1647
1648     def modifyChangelistUser(self, changelist, newUser):
1649         # fixup the user field of a changelist after it has been submitted.
1650         changes = p4CmdList("change -o %s" % changelist)
1651         if len(changes) != 1:
1652             die("Bad output from p4 change modifying %s to user %s" %
1653                 (changelist, newUser))
1654
1655         c = changes[0]
1656         if c['User'] == newUser: return   # nothing to do
1657         c['User'] = newUser
1658         input = marshal.dumps(c)
1659
1660         result = p4CmdList("change -f -i", stdin=input)
1661         for r in result:
1662             if 'code' in r:
1663                 if r['code'] == 'error':
1664                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1665             if 'data' in r:
1666                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1667                 return
1668         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1669
1670     def canChangeChangelists(self):
1671         # check to see if we have p4 admin or super-user permissions, either of
1672         # which are required to modify changelists.
1673         results = p4CmdList(["protects", self.depotPath])
1674         for r in results:
1675             if 'perm' in r:
1676                 if r['perm'] == 'admin':
1677                     return 1
1678                 if r['perm'] == 'super':
1679                     return 1
1680         return 0
1681
1682     def prepareSubmitTemplate(self, changelist=None):
1683         """Run "p4 change -o" to grab a change specification template.
1684            This does not use "p4 -G", as it is nice to keep the submission
1685            template in original order, since a human might edit it.
1686
1687            Remove lines in the Files section that show changes to files
1688            outside the depot path we're committing into."""
1689
1690         [upstream, settings] = findUpstreamBranchPoint()
1691
1692         template = """\
1693 # A Perforce Change Specification.
1694 #
1695 #  Change:      The change number. 'new' on a new changelist.
1696 #  Date:        The date this specification was last modified.
1697 #  Client:      The client on which the changelist was created.  Read-only.
1698 #  User:        The user who created the changelist.
1699 #  Status:      Either 'pending' or 'submitted'. Read-only.
1700 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1701 #  Description: Comments about the changelist.  Required.
1702 #  Jobs:        What opened jobs are to be closed by this changelist.
1703 #               You may delete jobs from this list.  (New changelists only.)
1704 #  Files:       What opened files from the default changelist are to be added
1705 #               to this changelist.  You may delete files from this list.
1706 #               (New changelists only.)
1707 """
1708         files_list = []
1709         inFilesSection = False
1710         change_entry = None
1711         args = ['change', '-o']
1712         if changelist:
1713             args.append(str(changelist))
1714         for entry in p4CmdList(args):
1715             if 'code' not in entry:
1716                 continue
1717             if entry['code'] == 'stat':
1718                 change_entry = entry
1719                 break
1720         if not change_entry:
1721             die('Failed to decode output of p4 change -o')
1722         for key, value in change_entry.iteritems():
1723             if key.startswith('File'):
1724                 if 'depot-paths' in settings:
1725                     if not [p for p in settings['depot-paths']
1726                             if p4PathStartsWith(value, p)]:
1727                         continue
1728                 else:
1729                     if not p4PathStartsWith(value, self.depotPath):
1730                         continue
1731                 files_list.append(value)
1732                 continue
1733         # Output in the order expected by prepareLogMessage
1734         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1735             if key not in change_entry:
1736                 continue
1737             template += '\n'
1738             template += key + ':'
1739             if key == 'Description':
1740                 template += '\n'
1741             for field_line in change_entry[key].splitlines():
1742                 template += '\t'+field_line+'\n'
1743         if len(files_list) > 0:
1744             template += '\n'
1745             template += 'Files:\n'
1746         for path in files_list:
1747             template += '\t'+path+'\n'
1748         return template
1749
1750     def edit_template(self, template_file):
1751         """Invoke the editor to let the user change the submission
1752            message.  Return true if okay to continue with the submit."""
1753
1754         # if configured to skip the editing part, just submit
1755         if gitConfigBool("git-p4.skipSubmitEdit"):
1756             return True
1757
1758         # look at the modification time, to check later if the user saved
1759         # the file
1760         mtime = os.stat(template_file).st_mtime
1761
1762         # invoke the editor
1763         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1764             editor = os.environ.get("P4EDITOR")
1765         else:
1766             editor = read_pipe("git var GIT_EDITOR").strip()
1767         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1768
1769         # If the file was not saved, prompt to see if this patch should
1770         # be skipped.  But skip this verification step if configured so.
1771         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1772             return True
1773
1774         # modification time updated means user saved the file
1775         if os.stat(template_file).st_mtime > mtime:
1776             return True
1777
1778         while True:
1779             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1780             if response == 'y':
1781                 return True
1782             if response == 'n':
1783                 return False
1784
1785     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1786         # diff
1787         if "P4DIFF" in os.environ:
1788             del(os.environ["P4DIFF"])
1789         diff = ""
1790         for editedFile in editedFiles:
1791             diff += p4_read_pipe(['diff', '-du',
1792                                   wildcard_encode(editedFile)])
1793
1794         # new file diff
1795         newdiff = ""
1796         for newFile in filesToAdd:
1797             newdiff += "==== new file ====\n"
1798             newdiff += "--- /dev/null\n"
1799             newdiff += "+++ %s\n" % newFile
1800
1801             is_link = os.path.islink(newFile)
1802             expect_link = newFile in symlinks
1803
1804             if is_link and expect_link:
1805                 newdiff += "+%s\n" % os.readlink(newFile)
1806             else:
1807                 f = open(newFile, "r")
1808                 for line in f.readlines():
1809                     newdiff += "+" + line
1810                 f.close()
1811
1812         return (diff + newdiff).replace('\r\n', '\n')
1813
1814     def applyCommit(self, id):
1815         """Apply one commit, return True if it succeeded."""
1816
1817         print("Applying", read_pipe(["git", "show", "-s",
1818                                      "--format=format:%h %s", id]))
1819
1820         (p4User, gitEmail) = self.p4UserForCommit(id)
1821
1822         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1823         filesToAdd = set()
1824         filesToChangeType = set()
1825         filesToDelete = set()
1826         editedFiles = set()
1827         pureRenameCopy = set()
1828         symlinks = set()
1829         filesToChangeExecBit = {}
1830         all_files = list()
1831
1832         for line in diff:
1833             diff = parseDiffTreeEntry(line)
1834             modifier = diff['status']
1835             path = diff['src']
1836             all_files.append(path)
1837
1838             if modifier == "M":
1839                 p4_edit(path)
1840                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1841                     filesToChangeExecBit[path] = diff['dst_mode']
1842                 editedFiles.add(path)
1843             elif modifier == "A":
1844                 filesToAdd.add(path)
1845                 filesToChangeExecBit[path] = diff['dst_mode']
1846                 if path in filesToDelete:
1847                     filesToDelete.remove(path)
1848
1849                 dst_mode = int(diff['dst_mode'], 8)
1850                 if dst_mode == 0o120000:
1851                     symlinks.add(path)
1852
1853             elif modifier == "D":
1854                 filesToDelete.add(path)
1855                 if path in filesToAdd:
1856                     filesToAdd.remove(path)
1857             elif modifier == "C":
1858                 src, dest = diff['src'], diff['dst']
1859                 p4_integrate(src, dest)
1860                 pureRenameCopy.add(dest)
1861                 if diff['src_sha1'] != diff['dst_sha1']:
1862                     p4_edit(dest)
1863                     pureRenameCopy.discard(dest)
1864                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1865                     p4_edit(dest)
1866                     pureRenameCopy.discard(dest)
1867                     filesToChangeExecBit[dest] = diff['dst_mode']
1868                 if self.isWindows:
1869                     # turn off read-only attribute
1870                     os.chmod(dest, stat.S_IWRITE)
1871                 os.unlink(dest)
1872                 editedFiles.add(dest)
1873             elif modifier == "R":
1874                 src, dest = diff['src'], diff['dst']
1875                 if self.p4HasMoveCommand:
1876                     p4_edit(src)        # src must be open before move
1877                     p4_move(src, dest)  # opens for (move/delete, move/add)
1878                 else:
1879                     p4_integrate(src, dest)
1880                     if diff['src_sha1'] != diff['dst_sha1']:
1881                         p4_edit(dest)
1882                     else:
1883                         pureRenameCopy.add(dest)
1884                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1885                     if not self.p4HasMoveCommand:
1886                         p4_edit(dest)   # with move: already open, writable
1887                     filesToChangeExecBit[dest] = diff['dst_mode']
1888                 if not self.p4HasMoveCommand:
1889                     if self.isWindows:
1890                         os.chmod(dest, stat.S_IWRITE)
1891                     os.unlink(dest)
1892                     filesToDelete.add(src)
1893                 editedFiles.add(dest)
1894             elif modifier == "T":
1895                 filesToChangeType.add(path)
1896             else:
1897                 die("unknown modifier %s for %s" % (modifier, path))
1898
1899         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1900         patchcmd = diffcmd + " | git apply "
1901         tryPatchCmd = patchcmd + "--check -"
1902         applyPatchCmd = patchcmd + "--check --apply -"
1903         patch_succeeded = True
1904
1905         if os.system(tryPatchCmd) != 0:
1906             fixed_rcs_keywords = False
1907             patch_succeeded = False
1908             print("Unfortunately applying the change failed!")
1909
1910             # Patch failed, maybe it's just RCS keyword woes. Look through
1911             # the patch to see if that's possible.
1912             if gitConfigBool("git-p4.attemptRCSCleanup"):
1913                 file = None
1914                 pattern = None
1915                 kwfiles = {}
1916                 for file in editedFiles | filesToDelete:
1917                     # did this file's delta contain RCS keywords?
1918                     pattern = p4_keywords_regexp_for_file(file)
1919
1920                     if pattern:
1921                         # this file is a possibility...look for RCS keywords.
1922                         regexp = re.compile(pattern, re.VERBOSE)
1923                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1924                             if regexp.search(line):
1925                                 if verbose:
1926                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1927                                 kwfiles[file] = pattern
1928                                 break
1929
1930                 for file in kwfiles:
1931                     if verbose:
1932                         print("zapping %s with %s" % (line,pattern))
1933                     # File is being deleted, so not open in p4.  Must
1934                     # disable the read-only bit on windows.
1935                     if self.isWindows and file not in editedFiles:
1936                         os.chmod(file, stat.S_IWRITE)
1937                     self.patchRCSKeywords(file, kwfiles[file])
1938                     fixed_rcs_keywords = True
1939
1940             if fixed_rcs_keywords:
1941                 print("Retrying the patch with RCS keywords cleaned up")
1942                 if os.system(tryPatchCmd) == 0:
1943                     patch_succeeded = True
1944
1945         if not patch_succeeded:
1946             for f in editedFiles:
1947                 p4_revert(f)
1948             return False
1949
1950         #
1951         # Apply the patch for real, and do add/delete/+x handling.
1952         #
1953         system(applyPatchCmd)
1954
1955         for f in filesToChangeType:
1956             p4_edit(f, "-t", "auto")
1957         for f in filesToAdd:
1958             p4_add(f)
1959         for f in filesToDelete:
1960             p4_revert(f)
1961             p4_delete(f)
1962
1963         # Set/clear executable bits
1964         for f in filesToChangeExecBit.keys():
1965             mode = filesToChangeExecBit[f]
1966             setP4ExecBit(f, mode)
1967
1968         update_shelve = 0
1969         if len(self.update_shelve) > 0:
1970             update_shelve = self.update_shelve.pop(0)
1971             p4_reopen_in_change(update_shelve, all_files)
1972
1973         #
1974         # Build p4 change description, starting with the contents
1975         # of the git commit message.
1976         #
1977         logMessage = extractLogMessageFromGitCommit(id)
1978         logMessage = logMessage.strip()
1979         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1980
1981         template = self.prepareSubmitTemplate(update_shelve)
1982         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1983
1984         if self.preserveUser:
1985            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1986
1987         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1988             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1989             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1990             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1991
1992         separatorLine = "######## everything below this line is just the diff #######\n"
1993         if not self.prepare_p4_only:
1994             submitTemplate += separatorLine
1995             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1996
1997         (handle, fileName) = tempfile.mkstemp()
1998         tmpFile = os.fdopen(handle, "w+b")
1999         if self.isWindows:
2000             submitTemplate = submitTemplate.replace("\n", "\r\n")
2001         tmpFile.write(submitTemplate)
2002         tmpFile.close()
2003
2004         if self.prepare_p4_only:
2005             #
2006             # Leave the p4 tree prepared, and the submit template around
2007             # and let the user decide what to do next
2008             #
2009             print()
2010             print("P4 workspace prepared for submission.")
2011             print("To submit or revert, go to client workspace")
2012             print("  " + self.clientPath)
2013             print()
2014             print("To submit, use \"p4 submit\" to write a new description,")
2015             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2016                   " \"git p4\"." % fileName)
2017             print("You can delete the file \"%s\" when finished." % fileName)
2018
2019             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2020                 print("To preserve change ownership by user %s, you must\n" \
2021                       "do \"p4 change -f <change>\" after submitting and\n" \
2022                       "edit the User field.")
2023             if pureRenameCopy:
2024                 print("After submitting, renamed files must be re-synced.")
2025                 print("Invoke \"p4 sync -f\" on each of these files:")
2026                 for f in pureRenameCopy:
2027                     print("  " + f)
2028
2029             print()
2030             print("To revert the changes, use \"p4 revert ...\", and delete")
2031             print("the submit template file \"%s\"" % fileName)
2032             if filesToAdd:
2033                 print("Since the commit adds new files, they must be deleted:")
2034                 for f in filesToAdd:
2035                     print("  " + f)
2036             print()
2037             return True
2038
2039         #
2040         # Let the user edit the change description, then submit it.
2041         #
2042         submitted = False
2043
2044         try:
2045             if self.edit_template(fileName):
2046                 # read the edited message and submit
2047                 tmpFile = open(fileName, "rb")
2048                 message = tmpFile.read()
2049                 tmpFile.close()
2050                 if self.isWindows:
2051                     message = message.replace("\r\n", "\n")
2052                 submitTemplate = message[:message.index(separatorLine)]
2053
2054                 if update_shelve:
2055                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2056                 elif self.shelve:
2057                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2058                 else:
2059                     p4_write_pipe(['submit', '-i'], submitTemplate)
2060                     # The rename/copy happened by applying a patch that created a
2061                     # new file.  This leaves it writable, which confuses p4.
2062                     for f in pureRenameCopy:
2063                         p4_sync(f, "-f")
2064
2065                 if self.preserveUser:
2066                     if p4User:
2067                         # Get last changelist number. Cannot easily get it from
2068                         # the submit command output as the output is
2069                         # unmarshalled.
2070                         changelist = self.lastP4Changelist()
2071                         self.modifyChangelistUser(changelist, p4User)
2072
2073                 submitted = True
2074
2075         finally:
2076             # skip this patch
2077             if not submitted or self.shelve:
2078                 if self.shelve:
2079                     print ("Reverting shelved files.")
2080                 else:
2081                     print ("Submission cancelled, undoing p4 changes.")
2082                 for f in editedFiles | filesToDelete:
2083                     p4_revert(f)
2084                 for f in filesToAdd:
2085                     p4_revert(f)
2086                     os.remove(f)
2087
2088         os.remove(fileName)
2089         return submitted
2090
2091     # Export git tags as p4 labels. Create a p4 label and then tag
2092     # with that.
2093     def exportGitTags(self, gitTags):
2094         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2095         if len(validLabelRegexp) == 0:
2096             validLabelRegexp = defaultLabelRegexp
2097         m = re.compile(validLabelRegexp)
2098
2099         for name in gitTags:
2100
2101             if not m.match(name):
2102                 if verbose:
2103                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2104                 continue
2105
2106             # Get the p4 commit this corresponds to
2107             logMessage = extractLogMessageFromGitCommit(name)
2108             values = extractSettingsGitLog(logMessage)
2109
2110             if 'change' not in values:
2111                 # a tag pointing to something not sent to p4; ignore
2112                 if verbose:
2113                     print("git tag %s does not give a p4 commit" % name)
2114                 continue
2115             else:
2116                 changelist = values['change']
2117
2118             # Get the tag details.
2119             inHeader = True
2120             isAnnotated = False
2121             body = []
2122             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2123                 l = l.strip()
2124                 if inHeader:
2125                     if re.match(r'tag\s+', l):
2126                         isAnnotated = True
2127                     elif re.match(r'\s*$', l):
2128                         inHeader = False
2129                         continue
2130                 else:
2131                     body.append(l)
2132
2133             if not isAnnotated:
2134                 body = ["lightweight tag imported by git p4\n"]
2135
2136             # Create the label - use the same view as the client spec we are using
2137             clientSpec = getClientSpec()
2138
2139             labelTemplate  = "Label: %s\n" % name
2140             labelTemplate += "Description:\n"
2141             for b in body:
2142                 labelTemplate += "\t" + b + "\n"
2143             labelTemplate += "View:\n"
2144             for depot_side in clientSpec.mappings:
2145                 labelTemplate += "\t%s\n" % depot_side
2146
2147             if self.dry_run:
2148                 print("Would create p4 label %s for tag" % name)
2149             elif self.prepare_p4_only:
2150                 print("Not creating p4 label %s for tag due to option" \
2151                       " --prepare-p4-only" % name)
2152             else:
2153                 p4_write_pipe(["label", "-i"], labelTemplate)
2154
2155                 # Use the label
2156                 p4_system(["tag", "-l", name] +
2157                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2158
2159                 if verbose:
2160                     print("created p4 label for tag %s" % name)
2161
2162     def run(self, args):
2163         if len(args) == 0:
2164             self.master = currentGitBranch()
2165         elif len(args) == 1:
2166             self.master = args[0]
2167             if not branchExists(self.master):
2168                 die("Branch %s does not exist" % self.master)
2169         else:
2170             return False
2171
2172         for i in self.update_shelve:
2173             if i <= 0:
2174                 sys.exit("invalid changelist %d" % i)
2175
2176         if self.master:
2177             allowSubmit = gitConfig("git-p4.allowSubmit")
2178             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2179                 die("%s is not in git-p4.allowSubmit" % self.master)
2180
2181         [upstream, settings] = findUpstreamBranchPoint()
2182         self.depotPath = settings['depot-paths'][0]
2183         if len(self.origin) == 0:
2184             self.origin = upstream
2185
2186         if len(self.update_shelve) > 0:
2187             self.shelve = True
2188
2189         if self.preserveUser:
2190             if not self.canChangeChangelists():
2191                 die("Cannot preserve user names without p4 super-user or admin permissions")
2192
2193         # if not set from the command line, try the config file
2194         if self.conflict_behavior is None:
2195             val = gitConfig("git-p4.conflict")
2196             if val:
2197                 if val not in self.conflict_behavior_choices:
2198                     die("Invalid value '%s' for config git-p4.conflict" % val)
2199             else:
2200                 val = "ask"
2201             self.conflict_behavior = val
2202
2203         if self.verbose:
2204             print("Origin branch is " + self.origin)
2205
2206         if len(self.depotPath) == 0:
2207             print("Internal error: cannot locate perforce depot path from existing branches")
2208             sys.exit(128)
2209
2210         self.useClientSpec = False
2211         if gitConfigBool("git-p4.useclientspec"):
2212             self.useClientSpec = True
2213         if self.useClientSpec:
2214             self.clientSpecDirs = getClientSpec()
2215
2216         # Check for the existence of P4 branches
2217         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2218
2219         if self.useClientSpec and not branchesDetected:
2220             # all files are relative to the client spec
2221             self.clientPath = getClientRoot()
2222         else:
2223             self.clientPath = p4Where(self.depotPath)
2224
2225         if self.clientPath == "":
2226             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2227
2228         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2229         self.oldWorkingDirectory = os.getcwd()
2230
2231         # ensure the clientPath exists
2232         new_client_dir = False
2233         if not os.path.exists(self.clientPath):
2234             new_client_dir = True
2235             os.makedirs(self.clientPath)
2236
2237         chdir(self.clientPath, is_client_path=True)
2238         if self.dry_run:
2239             print("Would synchronize p4 checkout in %s" % self.clientPath)
2240         else:
2241             print("Synchronizing p4 checkout...")
2242             if new_client_dir:
2243                 # old one was destroyed, and maybe nobody told p4
2244                 p4_sync("...", "-f")
2245             else:
2246                 p4_sync("...")
2247         self.check()
2248
2249         commits = []
2250         if self.master:
2251             committish = self.master
2252         else:
2253             committish = 'HEAD'
2254
2255         if self.commit != "":
2256             if self.commit.find("..") != -1:
2257                 limits_ish = self.commit.split("..")
2258                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2259                     commits.append(line.strip())
2260                 commits.reverse()
2261             else:
2262                 commits.append(self.commit)
2263         else:
2264             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2265                 commits.append(line.strip())
2266             commits.reverse()
2267
2268         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2269             self.checkAuthorship = False
2270         else:
2271             self.checkAuthorship = True
2272
2273         if self.preserveUser:
2274             self.checkValidP4Users(commits)
2275
2276         #
2277         # Build up a set of options to be passed to diff when
2278         # submitting each commit to p4.
2279         #
2280         if self.detectRenames:
2281             # command-line -M arg
2282             self.diffOpts = "-M"
2283         else:
2284             # If not explicitly set check the config variable
2285             detectRenames = gitConfig("git-p4.detectRenames")
2286
2287             if detectRenames.lower() == "false" or detectRenames == "":
2288                 self.diffOpts = ""
2289             elif detectRenames.lower() == "true":
2290                 self.diffOpts = "-M"
2291             else:
2292                 self.diffOpts = "-M%s" % detectRenames
2293
2294         # no command-line arg for -C or --find-copies-harder, just
2295         # config variables
2296         detectCopies = gitConfig("git-p4.detectCopies")
2297         if detectCopies.lower() == "false" or detectCopies == "":
2298             pass
2299         elif detectCopies.lower() == "true":
2300             self.diffOpts += " -C"
2301         else:
2302             self.diffOpts += " -C%s" % detectCopies
2303
2304         if gitConfigBool("git-p4.detectCopiesHarder"):
2305             self.diffOpts += " --find-copies-harder"
2306
2307         num_shelves = len(self.update_shelve)
2308         if num_shelves > 0 and num_shelves != len(commits):
2309             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2310                      (len(commits), num_shelves))
2311
2312         hooks_path = gitConfig("core.hooksPath")
2313         if len(hooks_path) <= 0:
2314             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2315
2316         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2317         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2318             sys.exit(1)
2319
2320         #
2321         # Apply the commits, one at a time.  On failure, ask if should
2322         # continue to try the rest of the patches, or quit.
2323         #
2324         if self.dry_run:
2325             print("Would apply")
2326         applied = []
2327         last = len(commits) - 1
2328         for i, commit in enumerate(commits):
2329             if self.dry_run:
2330                 print(" ", read_pipe(["git", "show", "-s",
2331                                       "--format=format:%h %s", commit]))
2332                 ok = True
2333             else:
2334                 ok = self.applyCommit(commit)
2335             if ok:
2336                 applied.append(commit)
2337             else:
2338                 if self.prepare_p4_only and i < last:
2339                     print("Processing only the first commit due to option" \
2340                           " --prepare-p4-only")
2341                     break
2342                 if i < last:
2343                     quit = False
2344                     while True:
2345                         # prompt for what to do, or use the option/variable
2346                         if self.conflict_behavior == "ask":
2347                             print("What do you want to do?")
2348                             response = raw_input("[s]kip this commit but apply"
2349                                                  " the rest, or [q]uit? ")
2350                             if not response:
2351                                 continue
2352                         elif self.conflict_behavior == "skip":
2353                             response = "s"
2354                         elif self.conflict_behavior == "quit":
2355                             response = "q"
2356                         else:
2357                             die("Unknown conflict_behavior '%s'" %
2358                                 self.conflict_behavior)
2359
2360                         if response[0] == "s":
2361                             print("Skipping this commit, but applying the rest")
2362                             break
2363                         if response[0] == "q":
2364                             print("Quitting")
2365                             quit = True
2366                             break
2367                     if quit:
2368                         break
2369
2370         chdir(self.oldWorkingDirectory)
2371         shelved_applied = "shelved" if self.shelve else "applied"
2372         if self.dry_run:
2373             pass
2374         elif self.prepare_p4_only:
2375             pass
2376         elif len(commits) == len(applied):
2377             print("All commits {0}!".format(shelved_applied))
2378
2379             sync = P4Sync()
2380             if self.branch:
2381                 sync.branch = self.branch
2382             if self.disable_p4sync:
2383                 sync.sync_origin_only()
2384             else:
2385                 sync.run([])
2386
2387                 if not self.disable_rebase:
2388                     rebase = P4Rebase()
2389                     rebase.rebase()
2390
2391         else:
2392             if len(applied) == 0:
2393                 print("No commits {0}.".format(shelved_applied))
2394             else:
2395                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2396                 for c in commits:
2397                     if c in applied:
2398                         star = "*"
2399                     else:
2400                         star = " "
2401                     print(star, read_pipe(["git", "show", "-s",
2402                                            "--format=format:%h %s",  c]))
2403                 print("You will have to do 'git p4 sync' and rebase.")
2404
2405         if gitConfigBool("git-p4.exportLabels"):
2406             self.exportLabels = True
2407
2408         if self.exportLabels:
2409             p4Labels = getP4Labels(self.depotPath)
2410             gitTags = getGitTags()
2411
2412             missingGitTags = gitTags - p4Labels
2413             self.exportGitTags(missingGitTags)
2414
2415         # exit with error unless everything applied perfectly
2416         if len(commits) != len(applied):
2417                 sys.exit(1)
2418
2419         return True
2420
2421 class View(object):
2422     """Represent a p4 view ("p4 help views"), and map files in a
2423        repo according to the view."""
2424
2425     def __init__(self, client_name):
2426         self.mappings = []
2427         self.client_prefix = "//%s/" % client_name
2428         # cache results of "p4 where" to lookup client file locations
2429         self.client_spec_path_cache = {}
2430
2431     def append(self, view_line):
2432         """Parse a view line, splitting it into depot and client
2433            sides.  Append to self.mappings, preserving order.  This
2434            is only needed for tag creation."""
2435
2436         # Split the view line into exactly two words.  P4 enforces
2437         # structure on these lines that simplifies this quite a bit.
2438         #
2439         # Either or both words may be double-quoted.
2440         # Single quotes do not matter.
2441         # Double-quote marks cannot occur inside the words.
2442         # A + or - prefix is also inside the quotes.
2443         # There are no quotes unless they contain a space.
2444         # The line is already white-space stripped.
2445         # The two words are separated by a single space.
2446         #
2447         if view_line[0] == '"':
2448             # First word is double quoted.  Find its end.
2449             close_quote_index = view_line.find('"', 1)
2450             if close_quote_index <= 0:
2451                 die("No first-word closing quote found: %s" % view_line)
2452             depot_side = view_line[1:close_quote_index]
2453             # skip closing quote and space
2454             rhs_index = close_quote_index + 1 + 1
2455         else:
2456             space_index = view_line.find(" ")
2457             if space_index <= 0:
2458                 die("No word-splitting space found: %s" % view_line)
2459             depot_side = view_line[0:space_index]
2460             rhs_index = space_index + 1
2461
2462         # prefix + means overlay on previous mapping
2463         if depot_side.startswith("+"):
2464             depot_side = depot_side[1:]
2465
2466         # prefix - means exclude this path, leave out of mappings
2467         exclude = False
2468         if depot_side.startswith("-"):
2469             exclude = True
2470             depot_side = depot_side[1:]
2471
2472         if not exclude:
2473             self.mappings.append(depot_side)
2474
2475     def convert_client_path(self, clientFile):
2476         # chop off //client/ part to make it relative
2477         if not clientFile.startswith(self.client_prefix):
2478             die("No prefix '%s' on clientFile '%s'" %
2479                 (self.client_prefix, clientFile))
2480         return clientFile[len(self.client_prefix):]
2481
2482     def update_client_spec_path_cache(self, files):
2483         """ Caching file paths by "p4 where" batch query """
2484
2485         # List depot file paths exclude that already cached
2486         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2487
2488         if len(fileArgs) == 0:
2489             return  # All files in cache
2490
2491         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2492         for res in where_result:
2493             if "code" in res and res["code"] == "error":
2494                 # assume error is "... file(s) not in client view"
2495                 continue
2496             if "clientFile" not in res:
2497                 die("No clientFile in 'p4 where' output")
2498             if "unmap" in res:
2499                 # it will list all of them, but only one not unmap-ped
2500                 continue
2501             if gitConfigBool("core.ignorecase"):
2502                 res['depotFile'] = res['depotFile'].lower()
2503             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2504
2505         # not found files or unmap files set to ""
2506         for depotFile in fileArgs:
2507             if gitConfigBool("core.ignorecase"):
2508                 depotFile = depotFile.lower()
2509             if depotFile not in self.client_spec_path_cache:
2510                 self.client_spec_path_cache[depotFile] = ""
2511
2512     def map_in_client(self, depot_path):
2513         """Return the relative location in the client where this
2514            depot file should live.  Returns "" if the file should
2515            not be mapped in the client."""
2516
2517         if gitConfigBool("core.ignorecase"):
2518             depot_path = depot_path.lower()
2519
2520         if depot_path in self.client_spec_path_cache:
2521             return self.client_spec_path_cache[depot_path]
2522
2523         die( "Error: %s is not found in client spec path" % depot_path )
2524         return ""
2525
2526 class P4Sync(Command, P4UserMap):
2527     delete_actions = ( "delete", "move/delete", "purge" )
2528
2529     def __init__(self):
2530         Command.__init__(self)
2531         P4UserMap.__init__(self)
2532         self.options = [
2533                 optparse.make_option("--branch", dest="branch"),
2534                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2535                 optparse.make_option("--changesfile", dest="changesFile"),
2536                 optparse.make_option("--silent", dest="silent", action="store_true"),
2537                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2538                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2539                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2540                                      help="Import into refs/heads/ , not refs/remotes"),
2541                 optparse.make_option("--max-changes", dest="maxChanges",
2542                                      help="Maximum number of changes to import"),
2543                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2544                                      help="Internal block size to use when iteratively calling p4 changes"),
2545                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2546                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2547                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2548                                      help="Only sync files that are included in the Perforce Client Spec"),
2549                 optparse.make_option("-/", dest="cloneExclude",
2550                                      action="append", type="string",
2551                                      help="exclude depot path"),
2552         ]
2553         self.description = """Imports from Perforce into a git repository.\n
2554     example:
2555     //depot/my/project/ -- to import the current head
2556     //depot/my/project/@all -- to import everything
2557     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2558
2559     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2560
2561         self.usage += " //depot/path[@revRange]"
2562         self.silent = False
2563         self.createdBranches = set()
2564         self.committedChanges = set()
2565         self.branch = ""
2566         self.detectBranches = False
2567         self.detectLabels = False
2568         self.importLabels = False
2569         self.changesFile = ""
2570         self.syncWithOrigin = True
2571         self.importIntoRemotes = True
2572         self.maxChanges = ""
2573         self.changes_block_size = None
2574         self.keepRepoPath = False
2575         self.depotPaths = None
2576         self.p4BranchesInGit = []
2577         self.cloneExclude = []
2578         self.useClientSpec = False
2579         self.useClientSpec_from_options = False
2580         self.clientSpecDirs = None
2581         self.tempBranches = []
2582         self.tempBranchLocation = "refs/git-p4-tmp"
2583         self.largeFileSystem = None
2584         self.suppress_meta_comment = False
2585
2586         if gitConfig('git-p4.largeFileSystem'):
2587             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2588             self.largeFileSystem = largeFileSystemConstructor(
2589                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2590             )
2591
2592         if gitConfig("git-p4.syncFromOrigin") == "false":
2593             self.syncWithOrigin = False
2594
2595         self.depotPaths = []
2596         self.changeRange = ""
2597         self.previousDepotPaths = []
2598         self.hasOrigin = False
2599
2600         # map from branch depot path to parent branch
2601         self.knownBranches = {}
2602         self.initialParents = {}
2603
2604         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2605         self.labels = {}
2606
2607     # Force a checkpoint in fast-import and wait for it to finish
2608     def checkpoint(self):
2609         self.gitStream.write("checkpoint\n\n")
2610         self.gitStream.write("progress checkpoint\n\n")
2611         out = self.gitOutput.readline()
2612         if self.verbose:
2613             print("checkpoint finished: " + out)
2614
2615     def cmp_shelved(self, path, filerev, revision):
2616         """ Determine if a path at revision #filerev is the same as the file
2617             at revision @revision for a shelved changelist. If they don't match,
2618             unshelving won't be safe (we will get other changes mixed in).
2619
2620             This is comparing the revision that the shelved changelist is *based* on, not
2621             the shelved changelist itself.
2622         """
2623         ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2624         if verbose:
2625             print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2626         return ret["status"] == "identical"
2627
2628     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
2629         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2630                              for path in self.cloneExclude]
2631         files = []
2632         fnum = 0
2633         while "depotFile%s" % fnum in commit:
2634             path =  commit["depotFile%s" % fnum]
2635
2636             if [p for p in self.cloneExclude
2637                 if p4PathStartsWith(path, p)]:
2638                 found = False
2639             else:
2640                 found = [p for p in self.depotPaths
2641                          if p4PathStartsWith(path, p)]
2642             if not found:
2643                 fnum = fnum + 1
2644                 continue
2645
2646             file = {}
2647             file["path"] = path
2648             file["rev"] = commit["rev%s" % fnum]
2649             file["action"] = commit["action%s" % fnum]
2650             file["type"] = commit["type%s" % fnum]
2651             if shelved:
2652                 file["shelved_cl"] = int(shelved_cl)
2653
2654                 # For shelved changelists, check that the revision of each file that the
2655                 # shelve was based on matches the revision that we are using for the
2656                 # starting point for git-fast-import (self.initialParent). Otherwise
2657                 # the resulting diff will contain deltas from multiple commits.
2658
2659                 if file["action"] != "add" and \
2660                     not self.cmp_shelved(path, file["rev"], origin_revision):
2661                     sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2662                         commit["change"], self.initialParent, path))
2663
2664             files.append(file)
2665             fnum = fnum + 1
2666         return files
2667
2668     def extractJobsFromCommit(self, commit):
2669         jobs = []
2670         jnum = 0
2671         while "job%s" % jnum in commit:
2672             job = commit["job%s" % jnum]
2673             jobs.append(job)
2674             jnum = jnum + 1
2675         return jobs
2676
2677     def stripRepoPath(self, path, prefixes):
2678         """When streaming files, this is called to map a p4 depot path
2679            to where it should go in git.  The prefixes are either
2680            self.depotPaths, or self.branchPrefixes in the case of
2681            branch detection."""
2682
2683         if self.useClientSpec:
2684             # branch detection moves files up a level (the branch name)
2685             # from what client spec interpretation gives
2686             path = self.clientSpecDirs.map_in_client(path)
2687             if self.detectBranches:
2688                 for b in self.knownBranches:
2689                     if path.startswith(b + "/"):
2690                         path = path[len(b)+1:]
2691
2692         elif self.keepRepoPath:
2693             # Preserve everything in relative path name except leading
2694             # //depot/; just look at first prefix as they all should
2695             # be in the same depot.
2696             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2697             if p4PathStartsWith(path, depot):
2698                 path = path[len(depot):]
2699
2700         else:
2701             for p in prefixes:
2702                 if p4PathStartsWith(path, p):
2703                     path = path[len(p):]
2704                     break
2705
2706         path = wildcard_decode(path)
2707         return path
2708
2709     def splitFilesIntoBranches(self, commit):
2710         """Look at each depotFile in the commit to figure out to what
2711            branch it belongs."""
2712
2713         if self.clientSpecDirs:
2714             files = self.extractFilesFromCommit(commit)
2715             self.clientSpecDirs.update_client_spec_path_cache(files)
2716
2717         branches = {}
2718         fnum = 0
2719         while "depotFile%s" % fnum in commit:
2720             path =  commit["depotFile%s" % fnum]
2721             found = [p for p in self.depotPaths
2722                      if p4PathStartsWith(path, p)]
2723             if not found:
2724                 fnum = fnum + 1
2725                 continue
2726
2727             file = {}
2728             file["path"] = path
2729             file["rev"] = commit["rev%s" % fnum]
2730             file["action"] = commit["action%s" % fnum]
2731             file["type"] = commit["type%s" % fnum]
2732             fnum = fnum + 1
2733
2734             # start with the full relative path where this file would
2735             # go in a p4 client
2736             if self.useClientSpec:
2737                 relPath = self.clientSpecDirs.map_in_client(path)
2738             else:
2739                 relPath = self.stripRepoPath(path, self.depotPaths)
2740
2741             for branch in self.knownBranches.keys():
2742                 # add a trailing slash so that a commit into qt/4.2foo
2743                 # doesn't end up in qt/4.2, e.g.
2744                 if relPath.startswith(branch + "/"):
2745                     if branch not in branches:
2746                         branches[branch] = []
2747                     branches[branch].append(file)
2748                     break
2749
2750         return branches
2751
2752     def writeToGitStream(self, gitMode, relPath, contents):
2753         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2754         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2755         for d in contents:
2756             self.gitStream.write(d)
2757         self.gitStream.write('\n')
2758
2759     def encodeWithUTF8(self, path):
2760         try:
2761             path.decode('ascii')
2762         except:
2763             encoding = 'utf8'
2764             if gitConfig('git-p4.pathEncoding'):
2765                 encoding = gitConfig('git-p4.pathEncoding')
2766             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2767             if self.verbose:
2768                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2769         return path
2770
2771     # output one file from the P4 stream
2772     # - helper for streamP4Files
2773
2774     def streamOneP4File(self, file, contents):
2775         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2776         relPath = self.encodeWithUTF8(relPath)
2777         if verbose:
2778             size = int(self.stream_file['fileSize'])
2779             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2780             sys.stdout.flush()
2781
2782         (type_base, type_mods) = split_p4_type(file["type"])
2783
2784         git_mode = "100644"
2785         if "x" in type_mods:
2786             git_mode = "100755"
2787         if type_base == "symlink":
2788             git_mode = "120000"
2789             # p4 print on a symlink sometimes contains "target\n";
2790             # if it does, remove the newline
2791             data = ''.join(contents)
2792             if not data:
2793                 # Some version of p4 allowed creating a symlink that pointed
2794                 # to nothing.  This causes p4 errors when checking out such
2795                 # a change, and errors here too.  Work around it by ignoring
2796                 # the bad symlink; hopefully a future change fixes it.
2797                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2798                 return
2799             elif data[-1] == '\n':
2800                 contents = [data[:-1]]
2801             else:
2802                 contents = [data]
2803
2804         if type_base == "utf16":
2805             # p4 delivers different text in the python output to -G
2806             # than it does when using "print -o", or normal p4 client
2807             # operations.  utf16 is converted to ascii or utf8, perhaps.
2808             # But ascii text saved as -t utf16 is completely mangled.
2809             # Invoke print -o to get the real contents.
2810             #
2811             # On windows, the newlines will always be mangled by print, so put
2812             # them back too.  This is not needed to the cygwin windows version,
2813             # just the native "NT" type.
2814             #
2815             try:
2816                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2817             except Exception as e:
2818                 if 'Translation of file content failed' in str(e):
2819                     type_base = 'binary'
2820                 else:
2821                     raise e
2822             else:
2823                 if p4_version_string().find('/NT') >= 0:
2824                     text = text.replace('\r\n', '\n')
2825                 contents = [ text ]
2826
2827         if type_base == "apple":
2828             # Apple filetype files will be streamed as a concatenation of
2829             # its appledouble header and the contents.  This is useless
2830             # on both macs and non-macs.  If using "print -q -o xx", it
2831             # will create "xx" with the data, and "%xx" with the header.
2832             # This is also not very useful.
2833             #
2834             # Ideally, someday, this script can learn how to generate
2835             # appledouble files directly and import those to git, but
2836             # non-mac machines can never find a use for apple filetype.
2837             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2838             return
2839
2840         # Note that we do not try to de-mangle keywords on utf16 files,
2841         # even though in theory somebody may want that.
2842         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2843         if pattern:
2844             regexp = re.compile(pattern, re.VERBOSE)
2845             text = ''.join(contents)
2846             text = regexp.sub(r'$\1$', text)
2847             contents = [ text ]
2848
2849         if self.largeFileSystem:
2850             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2851
2852         self.writeToGitStream(git_mode, relPath, contents)
2853
2854     def streamOneP4Deletion(self, file):
2855         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2856         relPath = self.encodeWithUTF8(relPath)
2857         if verbose:
2858             sys.stdout.write("delete %s\n" % relPath)
2859             sys.stdout.flush()
2860         self.gitStream.write("D %s\n" % relPath)
2861
2862         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2863             self.largeFileSystem.removeLargeFile(relPath)
2864
2865     # handle another chunk of streaming data
2866     def streamP4FilesCb(self, marshalled):
2867
2868         # catch p4 errors and complain
2869         err = None
2870         if "code" in marshalled:
2871             if marshalled["code"] == "error":
2872                 if "data" in marshalled:
2873                     err = marshalled["data"].rstrip()
2874
2875         if not err and 'fileSize' in self.stream_file:
2876             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2877             if required_bytes > 0:
2878                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2879                     os.getcwd(), required_bytes/1024/1024
2880                 )
2881
2882         if err:
2883             f = None
2884             if self.stream_have_file_info:
2885                 if "depotFile" in self.stream_file:
2886                     f = self.stream_file["depotFile"]
2887             # force a failure in fast-import, else an empty
2888             # commit will be made
2889             self.gitStream.write("\n")
2890             self.gitStream.write("die-now\n")
2891             self.gitStream.close()
2892             # ignore errors, but make sure it exits first
2893             self.importProcess.wait()
2894             if f:
2895                 die("Error from p4 print for %s: %s" % (f, err))
2896             else:
2897                 die("Error from p4 print: %s" % err)
2898
2899         if 'depotFile' in marshalled and self.stream_have_file_info:
2900             # start of a new file - output the old one first
2901             self.streamOneP4File(self.stream_file, self.stream_contents)
2902             self.stream_file = {}
2903             self.stream_contents = []
2904             self.stream_have_file_info = False
2905
2906         # pick up the new file information... for the
2907         # 'data' field we need to append to our array
2908         for k in marshalled.keys():
2909             if k == 'data':
2910                 if 'streamContentSize' not in self.stream_file:
2911                     self.stream_file['streamContentSize'] = 0
2912                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2913                 self.stream_contents.append(marshalled['data'])
2914             else:
2915                 self.stream_file[k] = marshalled[k]
2916
2917         if (verbose and
2918             'streamContentSize' in self.stream_file and
2919             'fileSize' in self.stream_file and
2920             'depotFile' in self.stream_file):
2921             size = int(self.stream_file["fileSize"])
2922             if size > 0:
2923                 progress = 100*self.stream_file['streamContentSize']/size
2924                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2925                 sys.stdout.flush()
2926
2927         self.stream_have_file_info = True
2928
2929     # Stream directly from "p4 files" into "git fast-import"
2930     def streamP4Files(self, files):
2931         filesForCommit = []
2932         filesToRead = []
2933         filesToDelete = []
2934
2935         for f in files:
2936             filesForCommit.append(f)
2937             if f['action'] in self.delete_actions:
2938                 filesToDelete.append(f)
2939             else:
2940                 filesToRead.append(f)
2941
2942         # deleted files...
2943         for f in filesToDelete:
2944             self.streamOneP4Deletion(f)
2945
2946         if len(filesToRead) > 0:
2947             self.stream_file = {}
2948             self.stream_contents = []
2949             self.stream_have_file_info = False
2950
2951             # curry self argument
2952             def streamP4FilesCbSelf(entry):
2953                 self.streamP4FilesCb(entry)
2954
2955             fileArgs = []
2956             for f in filesToRead:
2957                 if 'shelved_cl' in f:
2958                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2959                     # the contents
2960                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2961                 else:
2962                     fileArg = '%s#%s' % (f['path'], f['rev'])
2963
2964                 fileArgs.append(fileArg)
2965
2966             p4CmdList(["-x", "-", "print"],
2967                       stdin=fileArgs,
2968                       cb=streamP4FilesCbSelf)
2969
2970             # do the last chunk
2971             if 'depotFile' in self.stream_file:
2972                 self.streamOneP4File(self.stream_file, self.stream_contents)
2973
2974     def make_email(self, userid):
2975         if userid in self.users:
2976             return self.users[userid]
2977         else:
2978             return "%s <a@b>" % userid
2979
2980     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2981         """ Stream a p4 tag.
2982         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2983         """
2984
2985         if verbose:
2986             print("writing tag %s for commit %s" % (labelName, commit))
2987         gitStream.write("tag %s\n" % labelName)
2988         gitStream.write("from %s\n" % commit)
2989
2990         if 'Owner' in labelDetails:
2991             owner = labelDetails["Owner"]
2992         else:
2993             owner = None
2994
2995         # Try to use the owner of the p4 label, or failing that,
2996         # the current p4 user id.
2997         if owner:
2998             email = self.make_email(owner)
2999         else:
3000             email = self.make_email(self.p4UserId())
3001         tagger = "%s %s %s" % (email, epoch, self.tz)
3002
3003         gitStream.write("tagger %s\n" % tagger)
3004
3005         print("labelDetails=",labelDetails)
3006         if 'Description' in labelDetails:
3007             description = labelDetails['Description']
3008         else:
3009             description = 'Label from git p4'
3010
3011         gitStream.write("data %d\n" % len(description))
3012         gitStream.write(description)
3013         gitStream.write("\n")
3014
3015     def inClientSpec(self, path):
3016         if not self.clientSpecDirs:
3017             return True
3018         inClientSpec = self.clientSpecDirs.map_in_client(path)
3019         if not inClientSpec and self.verbose:
3020             print('Ignoring file outside of client spec: {0}'.format(path))
3021         return inClientSpec
3022
3023     def hasBranchPrefix(self, path):
3024         if not self.branchPrefixes:
3025             return True
3026         hasPrefix = [p for p in self.branchPrefixes
3027                         if p4PathStartsWith(path, p)]
3028         if not hasPrefix and self.verbose:
3029             print('Ignoring file outside of prefix: {0}'.format(path))
3030         return hasPrefix
3031
3032     def commit(self, details, files, branch, parent = ""):
3033         epoch = details["time"]
3034         author = details["user"]
3035         jobs = self.extractJobsFromCommit(details)
3036
3037         if self.verbose:
3038             print('commit into {0}'.format(branch))
3039
3040         if self.clientSpecDirs:
3041             self.clientSpecDirs.update_client_spec_path_cache(files)
3042
3043         files = [f for f in files
3044             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3045
3046         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
3047             print('Ignoring revision {0} as it would produce an empty commit.'
3048                 .format(details['change']))
3049             return
3050
3051         self.gitStream.write("commit %s\n" % branch)
3052         self.gitStream.write("mark :%s\n" % details["change"])
3053         self.committedChanges.add(int(details["change"]))
3054         committer = ""
3055         if author not in self.users:
3056             self.getUserMapFromPerforceServer()
3057         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3058
3059         self.gitStream.write("committer %s\n" % committer)
3060
3061         self.gitStream.write("data <<EOT\n")
3062         self.gitStream.write(details["desc"])
3063         if len(jobs) > 0:
3064             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3065
3066         if not self.suppress_meta_comment:
3067             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3068                                 (','.join(self.branchPrefixes), details["change"]))
3069             if len(details['options']) > 0:
3070                 self.gitStream.write(": options = %s" % details['options'])
3071             self.gitStream.write("]\n")
3072
3073         self.gitStream.write("EOT\n\n")
3074
3075         if len(parent) > 0:
3076             if self.verbose:
3077                 print("parent %s" % parent)
3078             self.gitStream.write("from %s\n" % parent)
3079
3080         self.streamP4Files(files)
3081         self.gitStream.write("\n")
3082
3083         change = int(details["change"])
3084
3085         if change in self.labels:
3086             label = self.labels[change]
3087             labelDetails = label[0]
3088             labelRevisions = label[1]
3089             if self.verbose:
3090                 print("Change %s is labelled %s" % (change, labelDetails))
3091
3092             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3093                                                 for p in self.branchPrefixes])
3094
3095             if len(files) == len(labelRevisions):
3096
3097                 cleanedFiles = {}
3098                 for info in files:
3099                     if info["action"] in self.delete_actions:
3100                         continue
3101                     cleanedFiles[info["depotFile"]] = info["rev"]
3102
3103                 if cleanedFiles == labelRevisions:
3104                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3105
3106                 else:
3107                     if not self.silent:
3108                         print("Tag %s does not match with change %s: files do not match."
3109                                % (labelDetails["label"], change))
3110
3111             else:
3112                 if not self.silent:
3113                     print("Tag %s does not match with change %s: file count is different."
3114                            % (labelDetails["label"], change))
3115
3116     # Build a dictionary of changelists and labels, for "detect-labels" option.
3117     def getLabels(self):
3118         self.labels = {}
3119
3120         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3121         if len(l) > 0 and not self.silent:
3122             print("Finding files belonging to labels in %s" % self.depotPaths)
3123
3124         for output in l:
3125             label = output["label"]
3126             revisions = {}
3127             newestChange = 0
3128             if self.verbose:
3129                 print("Querying files for label %s" % label)
3130             for file in p4CmdList(["files"] +
3131                                       ["%s...@%s" % (p, label)
3132                                           for p in self.depotPaths]):
3133                 revisions[file["depotFile"]] = file["rev"]
3134                 change = int(file["change"])
3135                 if change > newestChange:
3136                     newestChange = change
3137
3138             self.labels[newestChange] = [output, revisions]
3139
3140         if self.verbose:
3141             print("Label changes: %s" % self.labels.keys())
3142
3143     # Import p4 labels as git tags. A direct mapping does not
3144     # exist, so assume that if all the files are at the same revision
3145     # then we can use that, or it's something more complicated we should
3146     # just ignore.
3147     def importP4Labels(self, stream, p4Labels):
3148         if verbose:
3149             print("import p4 labels: " + ' '.join(p4Labels))
3150
3151         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3152         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3153         if len(validLabelRegexp) == 0:
3154             validLabelRegexp = defaultLabelRegexp
3155         m = re.compile(validLabelRegexp)
3156
3157         for name in p4Labels:
3158             commitFound = False
3159
3160             if not m.match(name):
3161                 if verbose:
3162                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3163                 continue
3164
3165             if name in ignoredP4Labels:
3166                 continue
3167
3168             labelDetails = p4CmdList(['label', "-o", name])[0]
3169
3170             # get the most recent changelist for each file in this label
3171             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3172                                 for p in self.depotPaths])
3173
3174             if 'change' in change:
3175                 # find the corresponding git commit; take the oldest commit
3176                 changelist = int(change['change'])
3177                 if changelist in self.committedChanges:
3178                     gitCommit = ":%d" % changelist       # use a fast-import mark
3179                     commitFound = True
3180                 else:
3181                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3182                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3183                     if len(gitCommit) == 0:
3184                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3185                     else:
3186                         commitFound = True
3187                         gitCommit = gitCommit.strip()
3188
3189                 if commitFound:
3190                     # Convert from p4 time format
3191                     try:
3192                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3193                     except ValueError:
3194                         print("Could not convert label time %s" % labelDetails['Update'])
3195                         tmwhen = 1
3196
3197                     when = int(time.mktime(tmwhen))
3198                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3199                     if verbose:
3200                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3201             else:
3202                 if verbose:
3203                     print("Label %s has no changelists - possibly deleted?" % name)
3204
3205             if not commitFound:
3206                 # We can't import this label; don't try again as it will get very
3207                 # expensive repeatedly fetching all the files for labels that will
3208                 # never be imported. If the label is moved in the future, the
3209                 # ignore will need to be removed manually.
3210                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3211
3212     def guessProjectName(self):
3213         for p in self.depotPaths:
3214             if p.endswith("/"):
3215                 p = p[:-1]
3216             p = p[p.strip().rfind("/") + 1:]
3217             if not p.endswith("/"):
3218                p += "/"
3219             return p
3220
3221     def getBranchMapping(self):
3222         lostAndFoundBranches = set()
3223
3224         user = gitConfig("git-p4.branchUser")
3225         if len(user) > 0:
3226             command = "branches -u %s" % user
3227         else:
3228             command = "branches"
3229
3230         for info in p4CmdList(command):
3231             details = p4Cmd(["branch", "-o", info["branch"]])
3232             viewIdx = 0
3233             while "View%s" % viewIdx in details:
3234                 paths = details["View%s" % viewIdx].split(" ")
3235                 viewIdx = viewIdx + 1
3236                 # require standard //depot/foo/... //depot/bar/... mapping
3237                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3238                     continue
3239                 source = paths[0]
3240                 destination = paths[1]
3241                 ## HACK
3242                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3243                     source = source[len(self.depotPaths[0]):-4]
3244                     destination = destination[len(self.depotPaths[0]):-4]
3245
3246                     if destination in self.knownBranches:
3247                         if not self.silent:
3248                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3249                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3250                         continue
3251
3252                     self.knownBranches[destination] = source
3253
3254                     lostAndFoundBranches.discard(destination)
3255
3256                     if source not in self.knownBranches:
3257                         lostAndFoundBranches.add(source)
3258
3259         # Perforce does not strictly require branches to be defined, so we also
3260         # check git config for a branch list.
3261         #
3262         # Example of branch definition in git config file:
3263         # [git-p4]
3264         #   branchList=main:branchA
3265         #   branchList=main:branchB
3266         #   branchList=branchA:branchC
3267         configBranches = gitConfigList("git-p4.branchList")
3268         for branch in configBranches:
3269             if branch:
3270                 (source, destination) = branch.split(":")
3271                 self.knownBranches[destination] = source
3272
3273                 lostAndFoundBranches.discard(destination)
3274
3275                 if source not in self.knownBranches:
3276                     lostAndFoundBranches.add(source)
3277
3278
3279         for branch in lostAndFoundBranches:
3280             self.knownBranches[branch] = branch
3281
3282     def getBranchMappingFromGitBranches(self):
3283         branches = p4BranchesInGit(self.importIntoRemotes)
3284         for branch in branches.keys():
3285             if branch == "master":
3286                 branch = "main"
3287             else:
3288                 branch = branch[len(self.projectName):]
3289             self.knownBranches[branch] = branch
3290
3291     def updateOptionDict(self, d):
3292         option_keys = {}
3293         if self.keepRepoPath:
3294             option_keys['keepRepoPath'] = 1
3295
3296         d["options"] = ' '.join(sorted(option_keys.keys()))
3297
3298     def readOptions(self, d):
3299         self.keepRepoPath = ('options' in d
3300                              and ('keepRepoPath' in d['options']))
3301
3302     def gitRefForBranch(self, branch):
3303         if branch == "main":
3304             return self.refPrefix + "master"
3305
3306         if len(branch) <= 0:
3307             return branch
3308
3309         return self.refPrefix + self.projectName + branch
3310
3311     def gitCommitByP4Change(self, ref, change):
3312         if self.verbose:
3313             print("looking in ref " + ref + " for change %s using bisect..." % change)
3314
3315         earliestCommit = ""
3316         latestCommit = parseRevision(ref)
3317
3318         while True:
3319             if self.verbose:
3320                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3321             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3322             if len(next) == 0:
3323                 if self.verbose:
3324                     print("argh")
3325                 return ""
3326             log = extractLogMessageFromGitCommit(next)
3327             settings = extractSettingsGitLog(log)
3328             currentChange = int(settings['change'])
3329             if self.verbose:
3330                 print("current change %s" % currentChange)
3331
3332             if currentChange == change:
3333                 if self.verbose:
3334                     print("found %s" % next)
3335                 return next
3336
3337             if currentChange < change:
3338                 earliestCommit = "^%s" % next
3339             else:
3340                 latestCommit = "%s" % next
3341
3342         return ""
3343
3344     def importNewBranch(self, branch, maxChange):
3345         # make fast-import flush all changes to disk and update the refs using the checkpoint
3346         # command so that we can try to find the branch parent in the git history
3347         self.gitStream.write("checkpoint\n\n");
3348         self.gitStream.flush();
3349         branchPrefix = self.depotPaths[0] + branch + "/"
3350         range = "@1,%s" % maxChange
3351         #print "prefix" + branchPrefix
3352         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3353         if len(changes) <= 0:
3354             return False
3355         firstChange = changes[0]
3356         #print "first change in branch: %s" % firstChange
3357         sourceBranch = self.knownBranches[branch]
3358         sourceDepotPath = self.depotPaths[0] + sourceBranch
3359         sourceRef = self.gitRefForBranch(sourceBranch)
3360         #print "source " + sourceBranch
3361
3362         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3363         #print "branch parent: %s" % branchParentChange
3364         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3365         if len(gitParent) > 0:
3366             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3367             #print "parent git commit: %s" % gitParent
3368
3369         self.importChanges(changes)
3370         return True
3371
3372     def searchParent(self, parent, branch, target):
3373         parentFound = False
3374         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3375                                      "--no-merges", parent]):
3376             blob = blob.strip()
3377             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3378                 parentFound = True
3379                 if self.verbose:
3380                     print("Found parent of %s in commit %s" % (branch, blob))
3381                 break
3382         if parentFound:
3383             return blob
3384         else:
3385             return None
3386
3387     def importChanges(self, changes, shelved=False, origin_revision=0):
3388         cnt = 1
3389         for change in changes:
3390             description = p4_describe(change, shelved)
3391             self.updateOptionDict(description)
3392
3393             if not self.silent:
3394                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3395                 sys.stdout.flush()
3396             cnt = cnt + 1
3397
3398             try:
3399                 if self.detectBranches:
3400                     branches = self.splitFilesIntoBranches(description)
3401                     for branch in branches.keys():
3402                         ## HACK  --hwn
3403                         branchPrefix = self.depotPaths[0] + branch + "/"
3404                         self.branchPrefixes = [ branchPrefix ]
3405
3406                         parent = ""
3407
3408                         filesForCommit = branches[branch]
3409
3410                         if self.verbose:
3411                             print("branch is %s" % branch)
3412
3413                         self.updatedBranches.add(branch)
3414
3415                         if branch not in self.createdBranches:
3416                             self.createdBranches.add(branch)
3417                             parent = self.knownBranches[branch]
3418                             if parent == branch:
3419                                 parent = ""
3420                             else:
3421                                 fullBranch = self.projectName + branch
3422                                 if fullBranch not in self.p4BranchesInGit:
3423                                     if not self.silent:
3424                                         print("\n    Importing new branch %s" % fullBranch);
3425                                     if self.importNewBranch(branch, change - 1):
3426                                         parent = ""
3427                                         self.p4BranchesInGit.append(fullBranch)
3428                                     if not self.silent:
3429                                         print("\n    Resuming with change %s" % change);
3430
3431                                 if self.verbose:
3432                                     print("parent determined through known branches: %s" % parent)
3433
3434                         branch = self.gitRefForBranch(branch)
3435                         parent = self.gitRefForBranch(parent)
3436
3437                         if self.verbose:
3438                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3439
3440                         if len(parent) == 0 and branch in self.initialParents:
3441                             parent = self.initialParents[branch]
3442                             del self.initialParents[branch]
3443
3444                         blob = None
3445                         if len(parent) > 0:
3446                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3447                             if self.verbose:
3448                                 print("Creating temporary branch: " + tempBranch)
3449                             self.commit(description, filesForCommit, tempBranch)
3450                             self.tempBranches.append(tempBranch)
3451                             self.checkpoint()
3452                             blob = self.searchParent(parent, branch, tempBranch)
3453                         if blob:
3454                             self.commit(description, filesForCommit, branch, blob)
3455                         else:
3456                             if self.verbose:
3457                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3458                             self.commit(description, filesForCommit, branch, parent)
3459                 else:
3460                     files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
3461                     self.commit(description, files, self.branch,
3462                                 self.initialParent)
3463                     # only needed once, to connect to the previous commit
3464                     self.initialParent = ""
3465             except IOError:
3466                 print(self.gitError.read())
3467                 sys.exit(1)
3468
3469     def sync_origin_only(self):
3470         if self.syncWithOrigin:
3471             self.hasOrigin = originP4BranchesExist()
3472             if self.hasOrigin:
3473                 if not self.silent:
3474                     print('Syncing with origin first, using "git fetch origin"')
3475                 system("git fetch origin")
3476
3477     def importHeadRevision(self, revision):
3478         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3479
3480         details = {}
3481         details["user"] = "git perforce import user"
3482         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3483                            % (' '.join(self.depotPaths), revision))
3484         details["change"] = revision
3485         newestRevision = 0
3486
3487         fileCnt = 0
3488         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3489
3490         for info in p4CmdList(["files"] + fileArgs):
3491
3492             if 'code' in info and info['code'] == 'error':
3493                 sys.stderr.write("p4 returned an error: %s\n"
3494                                  % info['data'])
3495                 if info['data'].find("must refer to client") >= 0:
3496                     sys.stderr.write("This particular p4 error is misleading.\n")
3497                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3498                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3499                 sys.exit(1)
3500             if 'p4ExitCode' in info:
3501                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3502                 sys.exit(1)
3503
3504
3505             change = int(info["change"])
3506             if change > newestRevision:
3507                 newestRevision = change
3508
3509             if info["action"] in self.delete_actions:
3510                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3511                 #fileCnt = fileCnt + 1
3512                 continue
3513
3514             for prop in ["depotFile", "rev", "action", "type" ]:
3515                 details["%s%s" % (prop, fileCnt)] = info[prop]
3516
3517             fileCnt = fileCnt + 1
3518
3519         details["change"] = newestRevision
3520
3521         # Use time from top-most change so that all git p4 clones of
3522         # the same p4 repo have the same commit SHA1s.
3523         res = p4_describe(newestRevision)
3524         details["time"] = res["time"]
3525
3526         self.updateOptionDict(details)
3527         try:
3528             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3529         except IOError:
3530             print("IO error with git fast-import. Is your git version recent enough?")
3531             print(self.gitError.read())
3532
3533     def openStreams(self):
3534         self.importProcess = subprocess.Popen(["git", "fast-import"],
3535                                               stdin=subprocess.PIPE,
3536                                               stdout=subprocess.PIPE,
3537                                               stderr=subprocess.PIPE);
3538         self.gitOutput = self.importProcess.stdout
3539         self.gitStream = self.importProcess.stdin
3540         self.gitError = self.importProcess.stderr
3541
3542     def closeStreams(self):
3543         self.gitStream.close()
3544         if self.importProcess.wait() != 0:
3545             die("fast-import failed: %s" % self.gitError.read())
3546         self.gitOutput.close()
3547         self.gitError.close()
3548
3549     def run(self, args):
3550         if self.importIntoRemotes:
3551             self.refPrefix = "refs/remotes/p4/"
3552         else:
3553             self.refPrefix = "refs/heads/p4/"
3554
3555         self.sync_origin_only()
3556
3557         branch_arg_given = bool(self.branch)
3558         if len(self.branch) == 0:
3559             self.branch = self.refPrefix + "master"
3560             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3561                 system("git update-ref %s refs/heads/p4" % self.branch)
3562                 system("git branch -D p4")
3563
3564         # accept either the command-line option, or the configuration variable
3565         if self.useClientSpec:
3566             # will use this after clone to set the variable
3567             self.useClientSpec_from_options = True
3568         else:
3569             if gitConfigBool("git-p4.useclientspec"):
3570                 self.useClientSpec = True
3571         if self.useClientSpec:
3572             self.clientSpecDirs = getClientSpec()
3573
3574         # TODO: should always look at previous commits,
3575         # merge with previous imports, if possible.
3576         if args == []:
3577             if self.hasOrigin:
3578                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3579
3580             # branches holds mapping from branch name to sha1
3581             branches = p4BranchesInGit(self.importIntoRemotes)
3582
3583             # restrict to just this one, disabling detect-branches
3584             if branch_arg_given:
3585                 short = self.branch.split("/")[-1]
3586                 if short in branches:
3587                     self.p4BranchesInGit = [ short ]
3588             else:
3589                 self.p4BranchesInGit = branches.keys()
3590
3591             if len(self.p4BranchesInGit) > 1:
3592                 if not self.silent:
3593                     print("Importing from/into multiple branches")
3594                 self.detectBranches = True
3595                 for branch in branches.keys():
3596                     self.initialParents[self.refPrefix + branch] = \
3597                         branches[branch]
3598
3599             if self.verbose:
3600                 print("branches: %s" % self.p4BranchesInGit)
3601
3602             p4Change = 0
3603             for branch in self.p4BranchesInGit:
3604                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3605
3606                 settings = extractSettingsGitLog(logMsg)
3607
3608                 self.readOptions(settings)
3609                 if ('depot-paths' in settings
3610                     and 'change' in settings):
3611                     change = int(settings['change']) + 1
3612                     p4Change = max(p4Change, change)
3613
3614                     depotPaths = sorted(settings['depot-paths'])
3615                     if self.previousDepotPaths == []:
3616                         self.previousDepotPaths = depotPaths
3617                     else:
3618                         paths = []
3619                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3620                             prev_list = prev.split("/")
3621                             cur_list = cur.split("/")
3622                             for i in range(0, min(len(cur_list), len(prev_list))):
3623                                 if cur_list[i] != prev_list[i]:
3624                                     i = i - 1
3625                                     break
3626
3627                             paths.append ("/".join(cur_list[:i + 1]))
3628
3629                         self.previousDepotPaths = paths
3630
3631             if p4Change > 0:
3632                 self.depotPaths = sorted(self.previousDepotPaths)
3633                 self.changeRange = "@%s,#head" % p4Change
3634                 if not self.silent and not self.detectBranches:
3635                     print("Performing incremental import into %s git branch" % self.branch)
3636
3637         # accept multiple ref name abbreviations:
3638         #    refs/foo/bar/branch -> use it exactly
3639         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3640         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3641         if not self.branch.startswith("refs/"):
3642             if self.importIntoRemotes:
3643                 prepend = "refs/remotes/"
3644             else:
3645                 prepend = "refs/heads/"
3646             if not self.branch.startswith("p4/"):
3647                 prepend += "p4/"
3648             self.branch = prepend + self.branch
3649
3650         if len(args) == 0 and self.depotPaths:
3651             if not self.silent:
3652                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3653         else:
3654             if self.depotPaths and self.depotPaths != args:
3655                 print("previous import used depot path %s and now %s was specified. "
3656                        "This doesn't work!" % (' '.join (self.depotPaths),
3657                                                ' '.join (args)))
3658                 sys.exit(1)
3659
3660             self.depotPaths = sorted(args)
3661
3662         revision = ""
3663         self.users = {}
3664
3665         # Make sure no revision specifiers are used when --changesfile
3666         # is specified.
3667         bad_changesfile = False
3668         if len(self.changesFile) > 0:
3669             for p in self.depotPaths:
3670                 if p.find("@") >= 0 or p.find("#") >= 0:
3671                     bad_changesfile = True
3672                     break
3673         if bad_changesfile:
3674             die("Option --changesfile is incompatible with revision specifiers")
3675
3676         newPaths = []
3677         for p in self.depotPaths:
3678             if p.find("@") != -1:
3679                 atIdx = p.index("@")
3680                 self.changeRange = p[atIdx:]
3681                 if self.changeRange == "@all":
3682                     self.changeRange = ""
3683                 elif ',' not in self.changeRange:
3684                     revision = self.changeRange
3685                     self.changeRange = ""
3686                 p = p[:atIdx]
3687             elif p.find("#") != -1:
3688                 hashIdx = p.index("#")
3689                 revision = p[hashIdx:]
3690                 p = p[:hashIdx]
3691             elif self.previousDepotPaths == []:
3692                 # pay attention to changesfile, if given, else import
3693                 # the entire p4 tree at the head revision
3694                 if len(self.changesFile) == 0:
3695                     revision = "#head"
3696
3697             p = re.sub ("\.\.\.$", "", p)
3698             if not p.endswith("/"):
3699                 p += "/"
3700
3701             newPaths.append(p)
3702
3703         self.depotPaths = newPaths
3704
3705         # --detect-branches may change this for each branch
3706         self.branchPrefixes = self.depotPaths
3707
3708         self.loadUserMapFromCache()
3709         self.labels = {}
3710         if self.detectLabels:
3711             self.getLabels();
3712
3713         if self.detectBranches:
3714             ## FIXME - what's a P4 projectName ?
3715             self.projectName = self.guessProjectName()
3716
3717             if self.hasOrigin:
3718                 self.getBranchMappingFromGitBranches()
3719             else:
3720                 self.getBranchMapping()
3721             if self.verbose:
3722                 print("p4-git branches: %s" % self.p4BranchesInGit)
3723                 print("initial parents: %s" % self.initialParents)
3724             for b in self.p4BranchesInGit:
3725                 if b != "master":
3726
3727                     ## FIXME
3728                     b = b[len(self.projectName):]
3729                 self.createdBranches.add(b)
3730
3731         self.openStreams()
3732
3733         if revision:
3734             self.importHeadRevision(revision)
3735         else:
3736             changes = []
3737
3738             if len(self.changesFile) > 0:
3739                 output = open(self.changesFile).readlines()
3740                 changeSet = set()
3741                 for line in output:
3742                     changeSet.add(int(line))
3743
3744                 for change in changeSet:
3745                     changes.append(change)
3746
3747                 changes.sort()
3748             else:
3749                 # catch "git p4 sync" with no new branches, in a repo that
3750                 # does not have any existing p4 branches
3751                 if len(args) == 0:
3752                     if not self.p4BranchesInGit:
3753                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3754
3755                     # The default branch is master, unless --branch is used to
3756                     # specify something else.  Make sure it exists, or complain
3757                     # nicely about how to use --branch.
3758                     if not self.detectBranches:
3759                         if not branch_exists(self.branch):
3760                             if branch_arg_given:
3761                                 die("Error: branch %s does not exist." % self.branch)
3762                             else:
3763                                 die("Error: no branch %s; perhaps specify one with --branch." %
3764                                     self.branch)
3765
3766                 if self.verbose:
3767                     print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3768                                                               self.changeRange))
3769                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3770
3771                 if len(self.maxChanges) > 0:
3772                     changes = changes[:min(int(self.maxChanges), len(changes))]
3773
3774             if len(changes) == 0:
3775                 if not self.silent:
3776                     print("No changes to import!")
3777             else:
3778                 if not self.silent and not self.detectBranches:
3779                     print("Import destination: %s" % self.branch)
3780
3781                 self.updatedBranches = set()
3782
3783                 if not self.detectBranches:
3784                     if args:
3785                         # start a new branch
3786                         self.initialParent = ""
3787                     else:
3788                         # build on a previous revision
3789                         self.initialParent = parseRevision(self.branch)
3790
3791                 self.importChanges(changes)
3792
3793                 if not self.silent:
3794                     print("")
3795                     if len(self.updatedBranches) > 0:
3796                         sys.stdout.write("Updated branches: ")
3797                         for b in self.updatedBranches:
3798                             sys.stdout.write("%s " % b)
3799                         sys.stdout.write("\n")
3800
3801         if gitConfigBool("git-p4.importLabels"):
3802             self.importLabels = True
3803
3804         if self.importLabels:
3805             p4Labels = getP4Labels(self.depotPaths)
3806             gitTags = getGitTags()
3807
3808             missingP4Labels = p4Labels - gitTags
3809             self.importP4Labels(self.gitStream, missingP4Labels)
3810
3811         self.closeStreams()
3812
3813         # Cleanup temporary branches created during import
3814         if self.tempBranches != []:
3815             for branch in self.tempBranches:
3816                 read_pipe("git update-ref -d %s" % branch)
3817             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3818
3819         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3820         # a convenient shortcut refname "p4".
3821         if self.importIntoRemotes:
3822             head_ref = self.refPrefix + "HEAD"
3823             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3824                 system(["git", "symbolic-ref", head_ref, self.branch])
3825
3826         return True
3827
3828 class P4Rebase(Command):
3829     def __init__(self):
3830         Command.__init__(self)
3831         self.options = [
3832                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3833         ]
3834         self.importLabels = False
3835         self.description = ("Fetches the latest revision from perforce and "
3836                             + "rebases the current work (branch) against it")
3837
3838     def run(self, args):
3839         sync = P4Sync()
3840         sync.importLabels = self.importLabels
3841         sync.run([])
3842
3843         return self.rebase()
3844
3845     def rebase(self):
3846         if os.system("git update-index --refresh") != 0:
3847             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3848         if len(read_pipe("git diff-index HEAD --")) > 0:
3849             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3850
3851         [upstream, settings] = findUpstreamBranchPoint()
3852         if len(upstream) == 0:
3853             die("Cannot find upstream branchpoint for rebase")
3854
3855         # the branchpoint may be p4/foo~3, so strip off the parent
3856         upstream = re.sub("~[0-9]+$", "", upstream)
3857
3858         print("Rebasing the current branch onto %s" % upstream)
3859         oldHead = read_pipe("git rev-parse HEAD").strip()
3860         system("git rebase %s" % upstream)
3861         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3862         return True
3863
3864 class P4Clone(P4Sync):
3865     def __init__(self):
3866         P4Sync.__init__(self)
3867         self.description = "Creates a new git repository and imports from Perforce into it"
3868         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3869         self.options += [
3870             optparse.make_option("--destination", dest="cloneDestination",
3871                                  action='store', default=None,
3872                                  help="where to leave result of the clone"),
3873             optparse.make_option("--bare", dest="cloneBare",
3874                                  action="store_true", default=False),
3875         ]
3876         self.cloneDestination = None
3877         self.needsGit = False
3878         self.cloneBare = False
3879
3880     def defaultDestination(self, args):
3881         ## TODO: use common prefix of args?
3882         depotPath = args[0]
3883         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3884         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3885         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3886         depotDir = re.sub(r"/$", "", depotDir)
3887         return os.path.split(depotDir)[1]
3888
3889     def run(self, args):
3890         if len(args) < 1:
3891             return False
3892
3893         if self.keepRepoPath and not self.cloneDestination:
3894             sys.stderr.write("Must specify destination for --keep-path\n")
3895             sys.exit(1)
3896
3897         depotPaths = args
3898
3899         if not self.cloneDestination and len(depotPaths) > 1:
3900             self.cloneDestination = depotPaths[-1]
3901             depotPaths = depotPaths[:-1]
3902
3903         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3904         for p in depotPaths:
3905             if not p.startswith("//"):
3906                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3907                 return False
3908
3909         if not self.cloneDestination:
3910             self.cloneDestination = self.defaultDestination(args)
3911
3912         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3913
3914         if not os.path.exists(self.cloneDestination):
3915             os.makedirs(self.cloneDestination)
3916         chdir(self.cloneDestination)
3917
3918         init_cmd = [ "git", "init" ]
3919         if self.cloneBare:
3920             init_cmd.append("--bare")
3921         retcode = subprocess.call(init_cmd)
3922         if retcode:
3923             raise CalledProcessError(retcode, init_cmd)
3924
3925         if not P4Sync.run(self, depotPaths):
3926             return False
3927
3928         # create a master branch and check out a work tree
3929         if gitBranchExists(self.branch):
3930             system([ "git", "branch", "master", self.branch ])
3931             if not self.cloneBare:
3932                 system([ "git", "checkout", "-f" ])
3933         else:
3934             print('Not checking out any branch, use ' \
3935                   '"git checkout -q -b master <branch>"')
3936
3937         # auto-set this variable if invoked with --use-client-spec
3938         if self.useClientSpec_from_options:
3939             system("git config --bool git-p4.useclientspec true")
3940
3941         return True
3942
3943 class P4Unshelve(Command):
3944     def __init__(self):
3945         Command.__init__(self)
3946         self.options = []
3947         self.origin = "HEAD"
3948         self.description = "Unshelve a P4 changelist into a git commit"
3949         self.usage = "usage: %prog [options] changelist"
3950         self.options += [
3951                 optparse.make_option("--origin", dest="origin",
3952                     help="Use this base revision instead of the default (%s)" % self.origin),
3953         ]
3954         self.verbose = False
3955         self.noCommit = False
3956         self.destbranch = "refs/remotes/p4/unshelved"
3957
3958     def renameBranch(self, branch_name):
3959         """ Rename the existing branch to branch_name.N
3960         """
3961
3962         found = True
3963         for i in range(0,1000):
3964             backup_branch_name = "{0}.{1}".format(branch_name, i)
3965             if not gitBranchExists(backup_branch_name):
3966                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3967                 gitDeleteRef(branch_name)
3968                 found = True
3969                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3970                 break
3971
3972         if not found:
3973             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3974
3975     def findLastP4Revision(self, starting_point):
3976         """ Look back from starting_point for the first commit created by git-p4
3977             to find the P4 commit we are based on, and the depot-paths.
3978         """
3979
3980         for parent in (range(65535)):
3981             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3982             settings = extractSettingsGitLog(log)
3983             if 'change' in settings:
3984                 return settings
3985
3986         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3987
3988     def run(self, args):
3989         if len(args) != 1:
3990             return False
3991
3992         if not gitBranchExists(self.origin):
3993             sys.exit("origin branch {0} does not exist".format(self.origin))
3994
3995         sync = P4Sync()
3996         changes = args
3997         sync.initialParent = self.origin
3998
3999         # use the first change in the list to construct the branch to unshelve into
4000         change = changes[0]
4001
4002         # if the target branch already exists, rename it
4003         branch_name = "{0}/{1}".format(self.destbranch, change)
4004         if gitBranchExists(branch_name):
4005             self.renameBranch(branch_name)
4006         sync.branch = branch_name
4007
4008         sync.verbose = self.verbose
4009         sync.suppress_meta_comment = True
4010
4011         settings = self.findLastP4Revision(self.origin)
4012         origin_revision = settings['change']
4013         sync.depotPaths = settings['depot-paths']
4014         sync.branchPrefixes = sync.depotPaths
4015
4016         sync.openStreams()
4017         sync.loadUserMapFromCache()
4018         sync.silent = True
4019         sync.importChanges(changes, shelved=True, origin_revision=origin_revision)
4020         sync.closeStreams()
4021
4022         print("unshelved changelist {0} into {1}".format(change, branch_name))
4023
4024         return True
4025
4026 class P4Branches(Command):
4027     def __init__(self):
4028         Command.__init__(self)
4029         self.options = [ ]
4030         self.description = ("Shows the git branches that hold imports and their "
4031                             + "corresponding perforce depot paths")
4032         self.verbose = False
4033
4034     def run(self, args):
4035         if originP4BranchesExist():
4036             createOrUpdateBranchesFromOrigin()
4037
4038         cmdline = "git rev-parse --symbolic "
4039         cmdline += " --remotes"
4040
4041         for line in read_pipe_lines(cmdline):
4042             line = line.strip()
4043
4044             if not line.startswith('p4/') or line == "p4/HEAD":
4045                 continue
4046             branch = line
4047
4048             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4049             settings = extractSettingsGitLog(log)
4050
4051             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4052         return True
4053
4054 class HelpFormatter(optparse.IndentedHelpFormatter):
4055     def __init__(self):
4056         optparse.IndentedHelpFormatter.__init__(self)
4057
4058     def format_description(self, description):
4059         if description:
4060             return description + "\n"
4061         else:
4062             return ""
4063
4064 def printUsage(commands):
4065     print("usage: %s <command> [options]" % sys.argv[0])
4066     print("")
4067     print("valid commands: %s" % ", ".join(commands))
4068     print("")
4069     print("Try %s <command> --help for command specific help." % sys.argv[0])
4070     print("")
4071
4072 commands = {
4073     "debug" : P4Debug,
4074     "submit" : P4Submit,
4075     "commit" : P4Submit,
4076     "sync" : P4Sync,
4077     "rebase" : P4Rebase,
4078     "clone" : P4Clone,
4079     "rollback" : P4RollBack,
4080     "branches" : P4Branches,
4081     "unshelve" : P4Unshelve,
4082 }
4083
4084
4085 def main():
4086     if len(sys.argv[1:]) == 0:
4087         printUsage(commands.keys())
4088         sys.exit(2)
4089
4090     cmdName = sys.argv[1]
4091     try:
4092         klass = commands[cmdName]
4093         cmd = klass()
4094     except KeyError:
4095         print("unknown command %s" % cmdName)
4096         print("")
4097         printUsage(commands.keys())
4098         sys.exit(2)
4099
4100     options = cmd.options
4101     cmd.gitdir = os.environ.get("GIT_DIR", None)
4102
4103     args = sys.argv[2:]
4104
4105     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4106     if cmd.needsGit:
4107         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4108
4109     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4110                                    options,
4111                                    description = cmd.description,
4112                                    formatter = HelpFormatter())
4113
4114     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4115     global verbose
4116     verbose = cmd.verbose
4117     if cmd.needsGit:
4118         if cmd.gitdir == None:
4119             cmd.gitdir = os.path.abspath(".git")
4120             if not isValidGitDir(cmd.gitdir):
4121                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4122                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4123                 if os.path.exists(cmd.gitdir):
4124                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4125                     if len(cdup) > 0:
4126                         chdir(cdup);
4127
4128         if not isValidGitDir(cmd.gitdir):
4129             if isValidGitDir(cmd.gitdir + "/.git"):
4130                 cmd.gitdir += "/.git"
4131             else:
4132                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4133
4134         # so git commands invoked from the P4 workspace will succeed
4135         os.environ["GIT_DIR"] = cmd.gitdir
4136
4137     if not cmd.run(args):
4138         parser.print_help()
4139         sys.exit(2)
4140
4141
4142 if __name__ == '__main__':
4143     main()