Merge branch 'js/commit-graph-chunk-table-fix'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 # support basestring in python3
31 try:
32     unicode = unicode
33 except NameError:
34     # 'unicode' is undefined, must be Python 3
35     str = str
36     unicode = str
37     bytes = bytes
38     basestring = (str,bytes)
39 else:
40     # 'unicode' exists, must be Python 2
41     str = str
42     unicode = unicode
43     bytes = str
44     basestring = basestring
45
46 try:
47     from subprocess import CalledProcessError
48 except ImportError:
49     # from python2.7:subprocess.py
50     # Exception classes used by this module.
51     class CalledProcessError(Exception):
52         """This exception is raised when a process run by check_call() returns
53         a non-zero exit status.  The exit status will be stored in the
54         returncode attribute."""
55         def __init__(self, returncode, cmd):
56             self.returncode = returncode
57             self.cmd = cmd
58         def __str__(self):
59             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
61 verbose = False
62
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
65
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
68
69 p4_access_checked = False
70
71 def p4_build_cmd(cmd):
72     """Build a suitable p4 command line.
73
74     This consolidates building and returning a p4 command line into one
75     location. It means that hooking into the environment, or other configuration
76     can be done more easily.
77     """
78     real_cmd = ["p4"]
79
80     user = gitConfig("git-p4.user")
81     if len(user) > 0:
82         real_cmd += ["-u",user]
83
84     password = gitConfig("git-p4.password")
85     if len(password) > 0:
86         real_cmd += ["-P", password]
87
88     port = gitConfig("git-p4.port")
89     if len(port) > 0:
90         real_cmd += ["-p", port]
91
92     host = gitConfig("git-p4.host")
93     if len(host) > 0:
94         real_cmd += ["-H", host]
95
96     client = gitConfig("git-p4.client")
97     if len(client) > 0:
98         real_cmd += ["-c", client]
99
100     retries = gitConfigInt("git-p4.retries")
101     if retries is None:
102         # Perform 3 retries by default
103         retries = 3
104     if retries > 0:
105         # Provide a way to not pass this option by setting git-p4.retries to 0
106         real_cmd += ["-r", str(retries)]
107
108     if isinstance(cmd,basestring):
109         real_cmd = ' '.join(real_cmd) + ' ' + cmd
110     else:
111         real_cmd += cmd
112
113     # now check that we can actually talk to the server
114     global p4_access_checked
115     if not p4_access_checked:
116         p4_access_checked = True    # suppress access checks in p4_check_access itself
117         p4_check_access()
118
119     return real_cmd
120
121 def git_dir(path):
122     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123         This won't automatically add ".git" to a directory.
124     """
125     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126     if not d or len(d) == 0:
127         return None
128     else:
129         return d
130
131 def chdir(path, is_client_path=False):
132     """Do chdir to the given path, and set the PWD environment
133        variable for use by P4.  It does not look at getcwd() output.
134        Since we're not using the shell, it is necessary to set the
135        PWD environment variable explicitly.
136
137        Normally, expand the path to force it to be absolute.  This
138        addresses the use of relative path names inside P4 settings,
139        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
140        as given; it looks for .p4config using PWD.
141
142        If is_client_path, the path was handed to us directly by p4,
143        and may be a symbolic link.  Do not call os.getcwd() in this
144        case, because it will cause p4 to think that PWD is not inside
145        the client path.
146        """
147
148     os.chdir(path)
149     if not is_client_path:
150         path = os.getcwd()
151     os.environ['PWD'] = path
152
153 def calcDiskFree():
154     """Return free space in bytes on the disk of the given dirname."""
155     if platform.system() == 'Windows':
156         free_bytes = ctypes.c_ulonglong(0)
157         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158         return free_bytes.value
159     else:
160         st = os.statvfs(os.getcwd())
161         return st.f_bavail * st.f_frsize
162
163 def die(msg):
164     if verbose:
165         raise Exception(msg)
166     else:
167         sys.stderr.write(msg + "\n")
168         sys.exit(1)
169
170 def write_pipe(c, stdin):
171     if verbose:
172         sys.stderr.write('Writing pipe: %s\n' % str(c))
173
174     expand = isinstance(c,basestring)
175     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176     pipe = p.stdin
177     val = pipe.write(stdin)
178     pipe.close()
179     if p.wait():
180         die('Command failed: %s' % str(c))
181
182     return val
183
184 def p4_write_pipe(c, stdin):
185     real_cmd = p4_build_cmd(c)
186     return write_pipe(real_cmd, stdin)
187
188 def read_pipe_full(c):
189     """ Read output from  command. Returns a tuple
190         of the return status, stdout text and stderr
191         text.
192     """
193     if verbose:
194         sys.stderr.write('Reading pipe: %s\n' % str(c))
195
196     expand = isinstance(c,basestring)
197     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198     (out, err) = p.communicate()
199     return (p.returncode, out, err)
200
201 def read_pipe(c, ignore_error=False):
202     """ Read output from  command. Returns the output text on
203         success. On failure, terminates execution, unless
204         ignore_error is True, when it returns an empty string.
205     """
206     (retcode, out, err) = read_pipe_full(c)
207     if retcode != 0:
208         if ignore_error:
209             out = ""
210         else:
211             die('Command failed: %s\nError: %s' % (str(c), err))
212     return out
213
214 def read_pipe_text(c):
215     """ Read output from a command with trailing whitespace stripped.
216         On error, returns None.
217     """
218     (retcode, out, err) = read_pipe_full(c)
219     if retcode != 0:
220         return None
221     else:
222         return out.rstrip()
223
224 def p4_read_pipe(c, ignore_error=False):
225     real_cmd = p4_build_cmd(c)
226     return read_pipe(real_cmd, ignore_error)
227
228 def read_pipe_lines(c):
229     if verbose:
230         sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232     expand = isinstance(c, basestring)
233     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234     pipe = p.stdout
235     val = pipe.readlines()
236     if pipe.close() or p.wait():
237         die('Command failed: %s' % str(c))
238
239     return val
240
241 def p4_read_pipe_lines(c):
242     """Specifically invoke p4 on the command supplied. """
243     real_cmd = p4_build_cmd(c)
244     return read_pipe_lines(real_cmd)
245
246 def p4_has_command(cmd):
247     """Ask p4 for help on this command.  If it returns an error, the
248        command does not exist in this version of p4."""
249     real_cmd = p4_build_cmd(["help", cmd])
250     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251                                    stderr=subprocess.PIPE)
252     p.communicate()
253     return p.returncode == 0
254
255 def p4_has_move_command():
256     """See if the move command exists, that it supports -k, and that
257        it has not been administratively disabled.  The arguments
258        must be correct, but the filenames do not have to exist.  Use
259        ones with wildcards so even if they exist, it will fail."""
260
261     if not p4_has_command("move"):
262         return False
263     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265     (out, err) = p.communicate()
266     # return code will be 1 in either case
267     if err.find("Invalid option") >= 0:
268         return False
269     if err.find("disabled") >= 0:
270         return False
271     # assume it failed because @... was invalid changelist
272     return True
273
274 def system(cmd, ignore_error=False):
275     expand = isinstance(cmd,basestring)
276     if verbose:
277         sys.stderr.write("executing %s\n" % str(cmd))
278     retcode = subprocess.call(cmd, shell=expand)
279     if retcode and not ignore_error:
280         raise CalledProcessError(retcode, cmd)
281
282     return retcode
283
284 def p4_system(cmd):
285     """Specifically invoke p4 as the system command. """
286     real_cmd = p4_build_cmd(cmd)
287     expand = isinstance(real_cmd, basestring)
288     retcode = subprocess.call(real_cmd, shell=expand)
289     if retcode:
290         raise CalledProcessError(retcode, real_cmd)
291
292 def die_bad_access(s):
293     die("failure accessing depot: {0}".format(s.rstrip()))
294
295 def p4_check_access(min_expiration=1):
296     """ Check if we can access Perforce - account still logged in
297     """
298     results = p4CmdList(["login", "-s"])
299
300     if len(results) == 0:
301         # should never get here: always get either some results, or a p4ExitCode
302         assert("could not parse response from perforce")
303
304     result = results[0]
305
306     if 'p4ExitCode' in result:
307         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308         die_bad_access("could not run p4")
309
310     code = result.get("code")
311     if not code:
312         # we get here if we couldn't connect and there was nothing to unmarshal
313         die_bad_access("could not connect")
314
315     elif code == "stat":
316         expiry = result.get("TicketExpiration")
317         if expiry:
318             expiry = int(expiry)
319             if expiry > min_expiration:
320                 # ok to carry on
321                 return
322             else:
323                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325         else:
326             # account without a timeout - all ok
327             return
328
329     elif code == "error":
330         data = result.get("data")
331         if data:
332             die_bad_access("p4 error: {0}".format(data))
333         else:
334             die_bad_access("unknown error")
335     elif code == "info":
336         return
337     else:
338         die_bad_access("unknown error code {0}".format(code))
339
340 _p4_version_string = None
341 def p4_version_string():
342     """Read the version string, showing just the last line, which
343        hopefully is the interesting version bit.
344
345        $ p4 -V
346        Perforce - The Fast Software Configuration Management System.
347        Copyright 1995-2011 Perforce Software.  All rights reserved.
348        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
349     """
350     global _p4_version_string
351     if not _p4_version_string:
352         a = p4_read_pipe_lines(["-V"])
353         _p4_version_string = a[-1].rstrip()
354     return _p4_version_string
355
356 def p4_integrate(src, dest):
357     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
358
359 def p4_sync(f, *options):
360     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
361
362 def p4_add(f):
363     # forcibly add file names with wildcards
364     if wildcard_present(f):
365         p4_system(["add", "-f", f])
366     else:
367         p4_system(["add", f])
368
369 def p4_delete(f):
370     p4_system(["delete", wildcard_encode(f)])
371
372 def p4_edit(f, *options):
373     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
374
375 def p4_revert(f):
376     p4_system(["revert", wildcard_encode(f)])
377
378 def p4_reopen(type, f):
379     p4_system(["reopen", "-t", type, wildcard_encode(f)])
380
381 def p4_reopen_in_change(changelist, files):
382     cmd = ["reopen", "-c", str(changelist)] + files
383     p4_system(cmd)
384
385 def p4_move(src, dest):
386     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
387
388 def p4_last_change():
389     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
390     return int(results[0]['change'])
391
392 def p4_describe(change, shelved=False):
393     """Make sure it returns a valid result by checking for
394        the presence of field "time".  Return a dict of the
395        results."""
396
397     cmd = ["describe", "-s"]
398     if shelved:
399         cmd += ["-S"]
400     cmd += [str(change)]
401
402     ds = p4CmdList(cmd, skip_info=True)
403     if len(ds) != 1:
404         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
405
406     d = ds[0]
407
408     if "p4ExitCode" in d:
409         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
410                                                       str(d)))
411     if "code" in d:
412         if d["code"] == "error":
413             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
414
415     if "time" not in d:
416         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
417
418     return d
419
420 #
421 # Canonicalize the p4 type and return a tuple of the
422 # base type, plus any modifiers.  See "p4 help filetypes"
423 # for a list and explanation.
424 #
425 def split_p4_type(p4type):
426
427     p4_filetypes_historical = {
428         "ctempobj": "binary+Sw",
429         "ctext": "text+C",
430         "cxtext": "text+Cx",
431         "ktext": "text+k",
432         "kxtext": "text+kx",
433         "ltext": "text+F",
434         "tempobj": "binary+FSw",
435         "ubinary": "binary+F",
436         "uresource": "resource+F",
437         "uxbinary": "binary+Fx",
438         "xbinary": "binary+x",
439         "xltext": "text+Fx",
440         "xtempobj": "binary+Swx",
441         "xtext": "text+x",
442         "xunicode": "unicode+x",
443         "xutf16": "utf16+x",
444     }
445     if p4type in p4_filetypes_historical:
446         p4type = p4_filetypes_historical[p4type]
447     mods = ""
448     s = p4type.split("+")
449     base = s[0]
450     mods = ""
451     if len(s) > 1:
452         mods = s[1]
453     return (base, mods)
454
455 #
456 # return the raw p4 type of a file (text, text+ko, etc)
457 #
458 def p4_type(f):
459     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
460     return results[0]['headType']
461
462 #
463 # Given a type base and modifier, return a regexp matching
464 # the keywords that can be expanded in the file
465 #
466 def p4_keywords_regexp_for_type(base, type_mods):
467     if base in ("text", "unicode", "binary"):
468         kwords = None
469         if "ko" in type_mods:
470             kwords = 'Id|Header'
471         elif "k" in type_mods:
472             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
473         else:
474             return None
475         pattern = r"""
476             \$              # Starts with a dollar, followed by...
477             (%s)            # one of the keywords, followed by...
478             (:[^$\n]+)?     # possibly an old expansion, followed by...
479             \$              # another dollar
480             """ % kwords
481         return pattern
482     else:
483         return None
484
485 #
486 # Given a file, return a regexp matching the possible
487 # RCS keywords that will be expanded, or None for files
488 # with kw expansion turned off.
489 #
490 def p4_keywords_regexp_for_file(file):
491     if not os.path.exists(file):
492         return None
493     else:
494         (type_base, type_mods) = split_p4_type(p4_type(file))
495         return p4_keywords_regexp_for_type(type_base, type_mods)
496
497 def setP4ExecBit(file, mode):
498     # Reopens an already open file and changes the execute bit to match
499     # the execute bit setting in the passed in mode.
500
501     p4Type = "+x"
502
503     if not isModeExec(mode):
504         p4Type = getP4OpenedType(file)
505         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
506         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
507         if p4Type[-1] == "+":
508             p4Type = p4Type[0:-1]
509
510     p4_reopen(p4Type, file)
511
512 def getP4OpenedType(file):
513     # Returns the perforce file type for the given file.
514
515     result = p4_read_pipe(["opened", wildcard_encode(file)])
516     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
517     if match:
518         return match.group(1)
519     else:
520         die("Could not determine file type for %s (result: '%s')" % (file, result))
521
522 # Return the set of all p4 labels
523 def getP4Labels(depotPaths):
524     labels = set()
525     if isinstance(depotPaths,basestring):
526         depotPaths = [depotPaths]
527
528     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
529         label = l['label']
530         labels.add(label)
531
532     return labels
533
534 # Return the set of all git tags
535 def getGitTags():
536     gitTags = set()
537     for line in read_pipe_lines(["git", "tag"]):
538         tag = line.strip()
539         gitTags.add(tag)
540     return gitTags
541
542 def diffTreePattern():
543     # This is a simple generator for the diff tree regex pattern. This could be
544     # a class variable if this and parseDiffTreeEntry were a part of a class.
545     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
546     while True:
547         yield pattern
548
549 def parseDiffTreeEntry(entry):
550     """Parses a single diff tree entry into its component elements.
551
552     See git-diff-tree(1) manpage for details about the format of the diff
553     output. This method returns a dictionary with the following elements:
554
555     src_mode - The mode of the source file
556     dst_mode - The mode of the destination file
557     src_sha1 - The sha1 for the source file
558     dst_sha1 - The sha1 fr the destination file
559     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
560     status_score - The score for the status (applicable for 'C' and 'R'
561                    statuses). This is None if there is no score.
562     src - The path for the source file.
563     dst - The path for the destination file. This is only present for
564           copy or renames. If it is not present, this is None.
565
566     If the pattern is not matched, None is returned."""
567
568     match = diffTreePattern().next().match(entry)
569     if match:
570         return {
571             'src_mode': match.group(1),
572             'dst_mode': match.group(2),
573             'src_sha1': match.group(3),
574             'dst_sha1': match.group(4),
575             'status': match.group(5),
576             'status_score': match.group(6),
577             'src': match.group(7),
578             'dst': match.group(10)
579         }
580     return None
581
582 def isModeExec(mode):
583     # Returns True if the given git mode represents an executable file,
584     # otherwise False.
585     return mode[-3:] == "755"
586
587 class P4Exception(Exception):
588     """ Base class for exceptions from the p4 client """
589     def __init__(self, exit_code):
590         self.p4ExitCode = exit_code
591
592 class P4ServerException(P4Exception):
593     """ Base class for exceptions where we get some kind of marshalled up result from the server """
594     def __init__(self, exit_code, p4_result):
595         super(P4ServerException, self).__init__(exit_code)
596         self.p4_result = p4_result
597         self.code = p4_result[0]['code']
598         self.data = p4_result[0]['data']
599
600 class P4RequestSizeException(P4ServerException):
601     """ One of the maxresults or maxscanrows errors """
602     def __init__(self, exit_code, p4_result, limit):
603         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
604         self.limit = limit
605
606 def isModeExecChanged(src_mode, dst_mode):
607     return isModeExec(src_mode) != isModeExec(dst_mode)
608
609 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
610         errors_as_exceptions=False):
611
612     if isinstance(cmd,basestring):
613         cmd = "-G " + cmd
614         expand = True
615     else:
616         cmd = ["-G"] + cmd
617         expand = False
618
619     cmd = p4_build_cmd(cmd)
620     if verbose:
621         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
622
623     # Use a temporary file to avoid deadlocks without
624     # subprocess.communicate(), which would put another copy
625     # of stdout into memory.
626     stdin_file = None
627     if stdin is not None:
628         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
629         if isinstance(stdin,basestring):
630             stdin_file.write(stdin)
631         else:
632             for i in stdin:
633                 stdin_file.write(i + '\n')
634         stdin_file.flush()
635         stdin_file.seek(0)
636
637     p4 = subprocess.Popen(cmd,
638                           shell=expand,
639                           stdin=stdin_file,
640                           stdout=subprocess.PIPE)
641
642     result = []
643     try:
644         while True:
645             entry = marshal.load(p4.stdout)
646             if skip_info:
647                 if 'code' in entry and entry['code'] == 'info':
648                     continue
649             if cb is not None:
650                 cb(entry)
651             else:
652                 result.append(entry)
653     except EOFError:
654         pass
655     exitCode = p4.wait()
656     if exitCode != 0:
657         if errors_as_exceptions:
658             if len(result) > 0:
659                 data = result[0].get('data')
660                 if data:
661                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
662                     if not m:
663                         m = re.search('Request too large \(over (\d+)\)', data)
664
665                     if m:
666                         limit = int(m.group(1))
667                         raise P4RequestSizeException(exitCode, result, limit)
668
669                 raise P4ServerException(exitCode, result)
670             else:
671                 raise P4Exception(exitCode)
672         else:
673             entry = {}
674             entry["p4ExitCode"] = exitCode
675             result.append(entry)
676
677     return result
678
679 def p4Cmd(cmd):
680     list = p4CmdList(cmd)
681     result = {}
682     for entry in list:
683         result.update(entry)
684     return result;
685
686 def p4Where(depotPath):
687     if not depotPath.endswith("/"):
688         depotPath += "/"
689     depotPathLong = depotPath + "..."
690     outputList = p4CmdList(["where", depotPathLong])
691     output = None
692     for entry in outputList:
693         if "depotFile" in entry:
694             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
695             # The base path always ends with "/...".
696             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
697                 output = entry
698                 break
699         elif "data" in entry:
700             data = entry.get("data")
701             space = data.find(" ")
702             if data[:space] == depotPath:
703                 output = entry
704                 break
705     if output == None:
706         return ""
707     if output["code"] == "error":
708         return ""
709     clientPath = ""
710     if "path" in output:
711         clientPath = output.get("path")
712     elif "data" in output:
713         data = output.get("data")
714         lastSpace = data.rfind(" ")
715         clientPath = data[lastSpace + 1:]
716
717     if clientPath.endswith("..."):
718         clientPath = clientPath[:-3]
719     return clientPath
720
721 def currentGitBranch():
722     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
723
724 def isValidGitDir(path):
725     return git_dir(path) != None
726
727 def parseRevision(ref):
728     return read_pipe("git rev-parse %s" % ref).strip()
729
730 def branchExists(ref):
731     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
732                      ignore_error=True)
733     return len(rev) > 0
734
735 def extractLogMessageFromGitCommit(commit):
736     logMessage = ""
737
738     ## fixme: title is first line of commit, not 1st paragraph.
739     foundTitle = False
740     for log in read_pipe_lines("git cat-file commit %s" % commit):
741        if not foundTitle:
742            if len(log) == 1:
743                foundTitle = True
744            continue
745
746        logMessage += log
747     return logMessage
748
749 def extractSettingsGitLog(log):
750     values = {}
751     for line in log.split("\n"):
752         line = line.strip()
753         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
754         if not m:
755             continue
756
757         assignments = m.group(1).split (':')
758         for a in assignments:
759             vals = a.split ('=')
760             key = vals[0].strip()
761             val = ('='.join (vals[1:])).strip()
762             if val.endswith ('\"') and val.startswith('"'):
763                 val = val[1:-1]
764
765             values[key] = val
766
767     paths = values.get("depot-paths")
768     if not paths:
769         paths = values.get("depot-path")
770     if paths:
771         values['depot-paths'] = paths.split(',')
772     return values
773
774 def gitBranchExists(branch):
775     proc = subprocess.Popen(["git", "rev-parse", branch],
776                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
777     return proc.wait() == 0;
778
779 def gitUpdateRef(ref, newvalue):
780     subprocess.check_call(["git", "update-ref", ref, newvalue])
781
782 def gitDeleteRef(ref):
783     subprocess.check_call(["git", "update-ref", "-d", ref])
784
785 _gitConfig = {}
786
787 def gitConfig(key, typeSpecifier=None):
788     if key not in _gitConfig:
789         cmd = [ "git", "config" ]
790         if typeSpecifier:
791             cmd += [ typeSpecifier ]
792         cmd += [ key ]
793         s = read_pipe(cmd, ignore_error=True)
794         _gitConfig[key] = s.strip()
795     return _gitConfig[key]
796
797 def gitConfigBool(key):
798     """Return a bool, using git config --bool.  It is True only if the
799        variable is set to true, and False if set to false or not present
800        in the config."""
801
802     if key not in _gitConfig:
803         _gitConfig[key] = gitConfig(key, '--bool') == "true"
804     return _gitConfig[key]
805
806 def gitConfigInt(key):
807     if key not in _gitConfig:
808         cmd = [ "git", "config", "--int", key ]
809         s = read_pipe(cmd, ignore_error=True)
810         v = s.strip()
811         try:
812             _gitConfig[key] = int(gitConfig(key, '--int'))
813         except ValueError:
814             _gitConfig[key] = None
815     return _gitConfig[key]
816
817 def gitConfigList(key):
818     if key not in _gitConfig:
819         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
820         _gitConfig[key] = s.strip().splitlines()
821         if _gitConfig[key] == ['']:
822             _gitConfig[key] = []
823     return _gitConfig[key]
824
825 def p4BranchesInGit(branchesAreInRemotes=True):
826     """Find all the branches whose names start with "p4/", looking
827        in remotes or heads as specified by the argument.  Return
828        a dictionary of { branch: revision } for each one found.
829        The branch names are the short names, without any
830        "p4/" prefix."""
831
832     branches = {}
833
834     cmdline = "git rev-parse --symbolic "
835     if branchesAreInRemotes:
836         cmdline += "--remotes"
837     else:
838         cmdline += "--branches"
839
840     for line in read_pipe_lines(cmdline):
841         line = line.strip()
842
843         # only import to p4/
844         if not line.startswith('p4/'):
845             continue
846         # special symbolic ref to p4/master
847         if line == "p4/HEAD":
848             continue
849
850         # strip off p4/ prefix
851         branch = line[len("p4/"):]
852
853         branches[branch] = parseRevision(line)
854
855     return branches
856
857 def branch_exists(branch):
858     """Make sure that the given ref name really exists."""
859
860     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
861     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
862     out, _ = p.communicate()
863     if p.returncode:
864         return False
865     # expect exactly one line of output: the branch name
866     return out.rstrip() == branch
867
868 def findUpstreamBranchPoint(head = "HEAD"):
869     branches = p4BranchesInGit()
870     # map from depot-path to branch name
871     branchByDepotPath = {}
872     for branch in branches.keys():
873         tip = branches[branch]
874         log = extractLogMessageFromGitCommit(tip)
875         settings = extractSettingsGitLog(log)
876         if "depot-paths" in settings:
877             paths = ",".join(settings["depot-paths"])
878             branchByDepotPath[paths] = "remotes/p4/" + branch
879
880     settings = None
881     parent = 0
882     while parent < 65535:
883         commit = head + "~%s" % parent
884         log = extractLogMessageFromGitCommit(commit)
885         settings = extractSettingsGitLog(log)
886         if "depot-paths" in settings:
887             paths = ",".join(settings["depot-paths"])
888             if paths in branchByDepotPath:
889                 return [branchByDepotPath[paths], settings]
890
891         parent = parent + 1
892
893     return ["", settings]
894
895 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
896     if not silent:
897         print("Creating/updating branch(es) in %s based on origin branch(es)"
898                % localRefPrefix)
899
900     originPrefix = "origin/p4/"
901
902     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
903         line = line.strip()
904         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
905             continue
906
907         headName = line[len(originPrefix):]
908         remoteHead = localRefPrefix + headName
909         originHead = line
910
911         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
912         if ('depot-paths' not in original
913             or 'change' not in original):
914             continue
915
916         update = False
917         if not gitBranchExists(remoteHead):
918             if verbose:
919                 print("creating %s" % remoteHead)
920             update = True
921         else:
922             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
923             if 'change' in settings:
924                 if settings['depot-paths'] == original['depot-paths']:
925                     originP4Change = int(original['change'])
926                     p4Change = int(settings['change'])
927                     if originP4Change > p4Change:
928                         print("%s (%s) is newer than %s (%s). "
929                                "Updating p4 branch from origin."
930                                % (originHead, originP4Change,
931                                   remoteHead, p4Change))
932                         update = True
933                 else:
934                     print("Ignoring: %s was imported from %s while "
935                            "%s was imported from %s"
936                            % (originHead, ','.join(original['depot-paths']),
937                               remoteHead, ','.join(settings['depot-paths'])))
938
939         if update:
940             system("git update-ref %s %s" % (remoteHead, originHead))
941
942 def originP4BranchesExist():
943         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
944
945
946 def p4ParseNumericChangeRange(parts):
947     changeStart = int(parts[0][1:])
948     if parts[1] == '#head':
949         changeEnd = p4_last_change()
950     else:
951         changeEnd = int(parts[1])
952
953     return (changeStart, changeEnd)
954
955 def chooseBlockSize(blockSize):
956     if blockSize:
957         return blockSize
958     else:
959         return defaultBlockSize
960
961 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
962     assert depotPaths
963
964     # Parse the change range into start and end. Try to find integer
965     # revision ranges as these can be broken up into blocks to avoid
966     # hitting server-side limits (maxrows, maxscanresults). But if
967     # that doesn't work, fall back to using the raw revision specifier
968     # strings, without using block mode.
969
970     if changeRange is None or changeRange == '':
971         changeStart = 1
972         changeEnd = p4_last_change()
973         block_size = chooseBlockSize(requestedBlockSize)
974     else:
975         parts = changeRange.split(',')
976         assert len(parts) == 2
977         try:
978             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
979             block_size = chooseBlockSize(requestedBlockSize)
980         except ValueError:
981             changeStart = parts[0][1:]
982             changeEnd = parts[1]
983             if requestedBlockSize:
984                 die("cannot use --changes-block-size with non-numeric revisions")
985             block_size = None
986
987     changes = set()
988
989     # Retrieve changes a block at a time, to prevent running
990     # into a MaxResults/MaxScanRows error from the server. If
991     # we _do_ hit one of those errors, turn down the block size
992
993     while True:
994         cmd = ['changes']
995
996         if block_size:
997             end = min(changeEnd, changeStart + block_size)
998             revisionRange = "%d,%d" % (changeStart, end)
999         else:
1000             revisionRange = "%s,%s" % (changeStart, changeEnd)
1001
1002         for p in depotPaths:
1003             cmd += ["%s...@%s" % (p, revisionRange)]
1004
1005         # fetch the changes
1006         try:
1007             result = p4CmdList(cmd, errors_as_exceptions=True)
1008         except P4RequestSizeException as e:
1009             if not block_size:
1010                 block_size = e.limit
1011             elif block_size > e.limit:
1012                 block_size = e.limit
1013             else:
1014                 block_size = max(2, block_size // 2)
1015
1016             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1017             continue
1018         except P4Exception as e:
1019             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1020
1021         # Insert changes in chronological order
1022         for entry in reversed(result):
1023             if 'change' not in entry:
1024                 continue
1025             changes.add(int(entry['change']))
1026
1027         if not block_size:
1028             break
1029
1030         if end >= changeEnd:
1031             break
1032
1033         changeStart = end + 1
1034
1035     changes = sorted(changes)
1036     return changes
1037
1038 def p4PathStartsWith(path, prefix):
1039     # This method tries to remedy a potential mixed-case issue:
1040     #
1041     # If UserA adds  //depot/DirA/file1
1042     # and UserB adds //depot/dira/file2
1043     #
1044     # we may or may not have a problem. If you have core.ignorecase=true,
1045     # we treat DirA and dira as the same directory
1046     if gitConfigBool("core.ignorecase"):
1047         return path.lower().startswith(prefix.lower())
1048     return path.startswith(prefix)
1049
1050 def getClientSpec():
1051     """Look at the p4 client spec, create a View() object that contains
1052        all the mappings, and return it."""
1053
1054     specList = p4CmdList("client -o")
1055     if len(specList) != 1:
1056         die('Output from "client -o" is %d lines, expecting 1' %
1057             len(specList))
1058
1059     # dictionary of all client parameters
1060     entry = specList[0]
1061
1062     # the //client/ name
1063     client_name = entry["Client"]
1064
1065     # just the keys that start with "View"
1066     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1067
1068     # hold this new View
1069     view = View(client_name)
1070
1071     # append the lines, in order, to the view
1072     for view_num in range(len(view_keys)):
1073         k = "View%d" % view_num
1074         if k not in view_keys:
1075             die("Expected view key %s missing" % k)
1076         view.append(entry[k])
1077
1078     return view
1079
1080 def getClientRoot():
1081     """Grab the client directory."""
1082
1083     output = p4CmdList("client -o")
1084     if len(output) != 1:
1085         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1086
1087     entry = output[0]
1088     if "Root" not in entry:
1089         die('Client has no "Root"')
1090
1091     return entry["Root"]
1092
1093 #
1094 # P4 wildcards are not allowed in filenames.  P4 complains
1095 # if you simply add them, but you can force it with "-f", in
1096 # which case it translates them into %xx encoding internally.
1097 #
1098 def wildcard_decode(path):
1099     # Search for and fix just these four characters.  Do % last so
1100     # that fixing it does not inadvertently create new %-escapes.
1101     # Cannot have * in a filename in windows; untested as to
1102     # what p4 would do in such a case.
1103     if not platform.system() == "Windows":
1104         path = path.replace("%2A", "*")
1105     path = path.replace("%23", "#") \
1106                .replace("%40", "@") \
1107                .replace("%25", "%")
1108     return path
1109
1110 def wildcard_encode(path):
1111     # do % first to avoid double-encoding the %s introduced here
1112     path = path.replace("%", "%25") \
1113                .replace("*", "%2A") \
1114                .replace("#", "%23") \
1115                .replace("@", "%40")
1116     return path
1117
1118 def wildcard_present(path):
1119     m = re.search("[*#@%]", path)
1120     return m is not None
1121
1122 class LargeFileSystem(object):
1123     """Base class for large file system support."""
1124
1125     def __init__(self, writeToGitStream):
1126         self.largeFiles = set()
1127         self.writeToGitStream = writeToGitStream
1128
1129     def generatePointer(self, cloneDestination, contentFile):
1130         """Return the content of a pointer file that is stored in Git instead of
1131            the actual content."""
1132         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1133
1134     def pushFile(self, localLargeFile):
1135         """Push the actual content which is not stored in the Git repository to
1136            a server."""
1137         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1138
1139     def hasLargeFileExtension(self, relPath):
1140         return reduce(
1141             lambda a, b: a or b,
1142             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1143             False
1144         )
1145
1146     def generateTempFile(self, contents):
1147         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1148         for d in contents:
1149             contentFile.write(d)
1150         contentFile.close()
1151         return contentFile.name
1152
1153     def exceedsLargeFileThreshold(self, relPath, contents):
1154         if gitConfigInt('git-p4.largeFileThreshold'):
1155             contentsSize = sum(len(d) for d in contents)
1156             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1157                 return True
1158         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159             contentsSize = sum(len(d) for d in contents)
1160             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1161                 return False
1162             contentTempFile = self.generateTempFile(contents)
1163             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1164             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1165             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1166             zf.close()
1167             compressedContentsSize = zf.infolist()[0].compress_size
1168             os.remove(contentTempFile)
1169             os.remove(compressedContentFile.name)
1170             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1171                 return True
1172         return False
1173
1174     def addLargeFile(self, relPath):
1175         self.largeFiles.add(relPath)
1176
1177     def removeLargeFile(self, relPath):
1178         self.largeFiles.remove(relPath)
1179
1180     def isLargeFile(self, relPath):
1181         return relPath in self.largeFiles
1182
1183     def processContent(self, git_mode, relPath, contents):
1184         """Processes the content of git fast import. This method decides if a
1185            file is stored in the large file system and handles all necessary
1186            steps."""
1187         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1188             contentTempFile = self.generateTempFile(contents)
1189             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1190             if pointer_git_mode:
1191                 git_mode = pointer_git_mode
1192             if localLargeFile:
1193                 # Move temp file to final location in large file system
1194                 largeFileDir = os.path.dirname(localLargeFile)
1195                 if not os.path.isdir(largeFileDir):
1196                     os.makedirs(largeFileDir)
1197                 shutil.move(contentTempFile, localLargeFile)
1198                 self.addLargeFile(relPath)
1199                 if gitConfigBool('git-p4.largeFilePush'):
1200                     self.pushFile(localLargeFile)
1201                 if verbose:
1202                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1203         return (git_mode, contents)
1204
1205 class MockLFS(LargeFileSystem):
1206     """Mock large file system for testing."""
1207
1208     def generatePointer(self, contentFile):
1209         """The pointer content is the original content prefixed with "pointer-".
1210            The local filename of the large file storage is derived from the file content.
1211            """
1212         with open(contentFile, 'r') as f:
1213             content = next(f)
1214             gitMode = '100644'
1215             pointerContents = 'pointer-' + content
1216             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1217             return (gitMode, pointerContents, localLargeFile)
1218
1219     def pushFile(self, localLargeFile):
1220         """The remote filename of the large file storage is the same as the local
1221            one but in a different directory.
1222            """
1223         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1224         if not os.path.exists(remotePath):
1225             os.makedirs(remotePath)
1226         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1227
1228 class GitLFS(LargeFileSystem):
1229     """Git LFS as backend for the git-p4 large file system.
1230        See https://git-lfs.github.com/ for details."""
1231
1232     def __init__(self, *args):
1233         LargeFileSystem.__init__(self, *args)
1234         self.baseGitAttributes = []
1235
1236     def generatePointer(self, contentFile):
1237         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1238            mode and content which is stored in the Git repository instead of
1239            the actual content. Return also the new location of the actual
1240            content.
1241            """
1242         if os.path.getsize(contentFile) == 0:
1243             return (None, '', None)
1244
1245         pointerProcess = subprocess.Popen(
1246             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1247             stdout=subprocess.PIPE
1248         )
1249         pointerFile = pointerProcess.stdout.read()
1250         if pointerProcess.wait():
1251             os.remove(contentFile)
1252             die('git-lfs pointer command failed. Did you install the extension?')
1253
1254         # Git LFS removed the preamble in the output of the 'pointer' command
1255         # starting from version 1.2.0. Check for the preamble here to support
1256         # earlier versions.
1257         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1258         if pointerFile.startswith('Git LFS pointer for'):
1259             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1260
1261         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1262         localLargeFile = os.path.join(
1263             os.getcwd(),
1264             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1265             oid,
1266         )
1267         # LFS Spec states that pointer files should not have the executable bit set.
1268         gitMode = '100644'
1269         return (gitMode, pointerFile, localLargeFile)
1270
1271     def pushFile(self, localLargeFile):
1272         uploadProcess = subprocess.Popen(
1273             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1274         )
1275         if uploadProcess.wait():
1276             die('git-lfs push command failed. Did you define a remote?')
1277
1278     def generateGitAttributes(self):
1279         return (
1280             self.baseGitAttributes +
1281             [
1282                 '\n',
1283                 '#\n',
1284                 '# Git LFS (see https://git-lfs.github.com/)\n',
1285                 '#\n',
1286             ] +
1287             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1288                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1289             ] +
1290             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1291                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1292             ]
1293         )
1294
1295     def addLargeFile(self, relPath):
1296         LargeFileSystem.addLargeFile(self, relPath)
1297         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1298
1299     def removeLargeFile(self, relPath):
1300         LargeFileSystem.removeLargeFile(self, relPath)
1301         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1302
1303     def processContent(self, git_mode, relPath, contents):
1304         if relPath == '.gitattributes':
1305             self.baseGitAttributes = contents
1306             return (git_mode, self.generateGitAttributes())
1307         else:
1308             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1309
1310 class Command:
1311     delete_actions = ( "delete", "move/delete", "purge" )
1312     add_actions = ( "add", "move/add" )
1313
1314     def __init__(self):
1315         self.usage = "usage: %prog [options]"
1316         self.needsGit = True
1317         self.verbose = False
1318
1319     # This is required for the "append" cloneExclude action
1320     def ensure_value(self, attr, value):
1321         if not hasattr(self, attr) or getattr(self, attr) is None:
1322             setattr(self, attr, value)
1323         return getattr(self, attr)
1324
1325 class P4UserMap:
1326     def __init__(self):
1327         self.userMapFromPerforceServer = False
1328         self.myP4UserId = None
1329
1330     def p4UserId(self):
1331         if self.myP4UserId:
1332             return self.myP4UserId
1333
1334         results = p4CmdList("user -o")
1335         for r in results:
1336             if 'User' in r:
1337                 self.myP4UserId = r['User']
1338                 return r['User']
1339         die("Could not find your p4 user id")
1340
1341     def p4UserIsMe(self, p4User):
1342         # return True if the given p4 user is actually me
1343         me = self.p4UserId()
1344         if not p4User or p4User != me:
1345             return False
1346         else:
1347             return True
1348
1349     def getUserCacheFilename(self):
1350         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1351         return home + "/.gitp4-usercache.txt"
1352
1353     def getUserMapFromPerforceServer(self):
1354         if self.userMapFromPerforceServer:
1355             return
1356         self.users = {}
1357         self.emails = {}
1358
1359         for output in p4CmdList("users"):
1360             if "User" not in output:
1361                 continue
1362             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1363             self.emails[output["Email"]] = output["User"]
1364
1365         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1366         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1367             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1368             if mapUser and len(mapUser[0]) == 3:
1369                 user = mapUser[0][0]
1370                 fullname = mapUser[0][1]
1371                 email = mapUser[0][2]
1372                 self.users[user] = fullname + " <" + email + ">"
1373                 self.emails[email] = user
1374
1375         s = ''
1376         for (key, val) in self.users.items():
1377             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1378
1379         open(self.getUserCacheFilename(), "wb").write(s)
1380         self.userMapFromPerforceServer = True
1381
1382     def loadUserMapFromCache(self):
1383         self.users = {}
1384         self.userMapFromPerforceServer = False
1385         try:
1386             cache = open(self.getUserCacheFilename(), "rb")
1387             lines = cache.readlines()
1388             cache.close()
1389             for line in lines:
1390                 entry = line.strip().split("\t")
1391                 self.users[entry[0]] = entry[1]
1392         except IOError:
1393             self.getUserMapFromPerforceServer()
1394
1395 class P4Debug(Command):
1396     def __init__(self):
1397         Command.__init__(self)
1398         self.options = []
1399         self.description = "A tool to debug the output of p4 -G."
1400         self.needsGit = False
1401
1402     def run(self, args):
1403         j = 0
1404         for output in p4CmdList(args):
1405             print('Element: %d' % j)
1406             j += 1
1407             print(output)
1408         return True
1409
1410 class P4RollBack(Command):
1411     def __init__(self):
1412         Command.__init__(self)
1413         self.options = [
1414             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1415         ]
1416         self.description = "A tool to debug the multi-branch import. Don't use :)"
1417         self.rollbackLocalBranches = False
1418
1419     def run(self, args):
1420         if len(args) != 1:
1421             return False
1422         maxChange = int(args[0])
1423
1424         if "p4ExitCode" in p4Cmd("changes -m 1"):
1425             die("Problems executing p4");
1426
1427         if self.rollbackLocalBranches:
1428             refPrefix = "refs/heads/"
1429             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1430         else:
1431             refPrefix = "refs/remotes/"
1432             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1433
1434         for line in lines:
1435             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1436                 line = line.strip()
1437                 ref = refPrefix + line
1438                 log = extractLogMessageFromGitCommit(ref)
1439                 settings = extractSettingsGitLog(log)
1440
1441                 depotPaths = settings['depot-paths']
1442                 change = settings['change']
1443
1444                 changed = False
1445
1446                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1447                                                            for p in depotPaths]))) == 0:
1448                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1449                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1450                     continue
1451
1452                 while change and int(change) > maxChange:
1453                     changed = True
1454                     if self.verbose:
1455                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1456                     system("git update-ref %s \"%s^\"" % (ref, ref))
1457                     log = extractLogMessageFromGitCommit(ref)
1458                     settings =  extractSettingsGitLog(log)
1459
1460
1461                     depotPaths = settings['depot-paths']
1462                     change = settings['change']
1463
1464                 if changed:
1465                     print("%s rewound to %s" % (ref, change))
1466
1467         return True
1468
1469 class P4Submit(Command, P4UserMap):
1470
1471     conflict_behavior_choices = ("ask", "skip", "quit")
1472
1473     def __init__(self):
1474         Command.__init__(self)
1475         P4UserMap.__init__(self)
1476         self.options = [
1477                 optparse.make_option("--origin", dest="origin"),
1478                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1479                 # preserve the user, requires relevant p4 permissions
1480                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1481                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1482                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1483                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1484                 optparse.make_option("--conflict", dest="conflict_behavior",
1485                                      choices=self.conflict_behavior_choices),
1486                 optparse.make_option("--branch", dest="branch"),
1487                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1488                                      help="Shelve instead of submit. Shelved files are reverted, "
1489                                      "restoring the workspace to the state before the shelve"),
1490                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1491                                      metavar="CHANGELIST",
1492                                      help="update an existing shelved changelist, implies --shelve, "
1493                                            "repeat in-order for multiple shelved changelists"),
1494                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1495                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1496                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1497                                      help="Disable rebase after submit is completed. Can be useful if you "
1498                                      "work from a local git branch that is not master"),
1499                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1500                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1501         ]
1502         self.description = """Submit changes from git to the perforce depot.\n
1503     The `p4-pre-submit` hook is executed if it exists and is executable.
1504     The hook takes no parameters and nothing from standard input. Exiting with
1505     non-zero status from this script prevents `git-p4 submit` from launching.
1506
1507     One usage scenario is to run unit tests in the hook."""
1508
1509         self.usage += " [name of git branch to submit into perforce depot]"
1510         self.origin = ""
1511         self.detectRenames = False
1512         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1513         self.dry_run = False
1514         self.shelve = False
1515         self.update_shelve = list()
1516         self.commit = ""
1517         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1518         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1519         self.prepare_p4_only = False
1520         self.conflict_behavior = None
1521         self.isWindows = (platform.system() == "Windows")
1522         self.exportLabels = False
1523         self.p4HasMoveCommand = p4_has_move_command()
1524         self.branch = None
1525
1526         if gitConfig('git-p4.largeFileSystem'):
1527             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1528
1529     def check(self):
1530         if len(p4CmdList("opened ...")) > 0:
1531             die("You have files opened with perforce! Close them before starting the sync.")
1532
1533     def separate_jobs_from_description(self, message):
1534         """Extract and return a possible Jobs field in the commit
1535            message.  It goes into a separate section in the p4 change
1536            specification.
1537
1538            A jobs line starts with "Jobs:" and looks like a new field
1539            in a form.  Values are white-space separated on the same
1540            line or on following lines that start with a tab.
1541
1542            This does not parse and extract the full git commit message
1543            like a p4 form.  It just sees the Jobs: line as a marker
1544            to pass everything from then on directly into the p4 form,
1545            but outside the description section.
1546
1547            Return a tuple (stripped log message, jobs string)."""
1548
1549         m = re.search(r'^Jobs:', message, re.MULTILINE)
1550         if m is None:
1551             return (message, None)
1552
1553         jobtext = message[m.start():]
1554         stripped_message = message[:m.start()].rstrip()
1555         return (stripped_message, jobtext)
1556
1557     def prepareLogMessage(self, template, message, jobs):
1558         """Edits the template returned from "p4 change -o" to insert
1559            the message in the Description field, and the jobs text in
1560            the Jobs field."""
1561         result = ""
1562
1563         inDescriptionSection = False
1564
1565         for line in template.split("\n"):
1566             if line.startswith("#"):
1567                 result += line + "\n"
1568                 continue
1569
1570             if inDescriptionSection:
1571                 if line.startswith("Files:") or line.startswith("Jobs:"):
1572                     inDescriptionSection = False
1573                     # insert Jobs section
1574                     if jobs:
1575                         result += jobs + "\n"
1576                 else:
1577                     continue
1578             else:
1579                 if line.startswith("Description:"):
1580                     inDescriptionSection = True
1581                     line += "\n"
1582                     for messageLine in message.split("\n"):
1583                         line += "\t" + messageLine + "\n"
1584
1585             result += line + "\n"
1586
1587         return result
1588
1589     def patchRCSKeywords(self, file, pattern):
1590         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1591         (handle, outFileName) = tempfile.mkstemp(dir='.')
1592         try:
1593             outFile = os.fdopen(handle, "w+")
1594             inFile = open(file, "r")
1595             regexp = re.compile(pattern, re.VERBOSE)
1596             for line in inFile.readlines():
1597                 line = regexp.sub(r'$\1$', line)
1598                 outFile.write(line)
1599             inFile.close()
1600             outFile.close()
1601             # Forcibly overwrite the original file
1602             os.unlink(file)
1603             shutil.move(outFileName, file)
1604         except:
1605             # cleanup our temporary file
1606             os.unlink(outFileName)
1607             print("Failed to strip RCS keywords in %s" % file)
1608             raise
1609
1610         print("Patched up RCS keywords in %s" % file)
1611
1612     def p4UserForCommit(self,id):
1613         # Return the tuple (perforce user,git email) for a given git commit id
1614         self.getUserMapFromPerforceServer()
1615         gitEmail = read_pipe(["git", "log", "--max-count=1",
1616                               "--format=%ae", id])
1617         gitEmail = gitEmail.strip()
1618         if gitEmail not in self.emails:
1619             return (None,gitEmail)
1620         else:
1621             return (self.emails[gitEmail],gitEmail)
1622
1623     def checkValidP4Users(self,commits):
1624         # check if any git authors cannot be mapped to p4 users
1625         for id in commits:
1626             (user,email) = self.p4UserForCommit(id)
1627             if not user:
1628                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1629                 if gitConfigBool("git-p4.allowMissingP4Users"):
1630                     print("%s" % msg)
1631                 else:
1632                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1633
1634     def lastP4Changelist(self):
1635         # Get back the last changelist number submitted in this client spec. This
1636         # then gets used to patch up the username in the change. If the same
1637         # client spec is being used by multiple processes then this might go
1638         # wrong.
1639         results = p4CmdList("client -o")        # find the current client
1640         client = None
1641         for r in results:
1642             if 'Client' in r:
1643                 client = r['Client']
1644                 break
1645         if not client:
1646             die("could not get client spec")
1647         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1648         for r in results:
1649             if 'change' in r:
1650                 return r['change']
1651         die("Could not get changelist number for last submit - cannot patch up user details")
1652
1653     def modifyChangelistUser(self, changelist, newUser):
1654         # fixup the user field of a changelist after it has been submitted.
1655         changes = p4CmdList("change -o %s" % changelist)
1656         if len(changes) != 1:
1657             die("Bad output from p4 change modifying %s to user %s" %
1658                 (changelist, newUser))
1659
1660         c = changes[0]
1661         if c['User'] == newUser: return   # nothing to do
1662         c['User'] = newUser
1663         input = marshal.dumps(c)
1664
1665         result = p4CmdList("change -f -i", stdin=input)
1666         for r in result:
1667             if 'code' in r:
1668                 if r['code'] == 'error':
1669                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1670             if 'data' in r:
1671                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1672                 return
1673         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1674
1675     def canChangeChangelists(self):
1676         # check to see if we have p4 admin or super-user permissions, either of
1677         # which are required to modify changelists.
1678         results = p4CmdList(["protects", self.depotPath])
1679         for r in results:
1680             if 'perm' in r:
1681                 if r['perm'] == 'admin':
1682                     return 1
1683                 if r['perm'] == 'super':
1684                     return 1
1685         return 0
1686
1687     def prepareSubmitTemplate(self, changelist=None):
1688         """Run "p4 change -o" to grab a change specification template.
1689            This does not use "p4 -G", as it is nice to keep the submission
1690            template in original order, since a human might edit it.
1691
1692            Remove lines in the Files section that show changes to files
1693            outside the depot path we're committing into."""
1694
1695         [upstream, settings] = findUpstreamBranchPoint()
1696
1697         template = """\
1698 # A Perforce Change Specification.
1699 #
1700 #  Change:      The change number. 'new' on a new changelist.
1701 #  Date:        The date this specification was last modified.
1702 #  Client:      The client on which the changelist was created.  Read-only.
1703 #  User:        The user who created the changelist.
1704 #  Status:      Either 'pending' or 'submitted'. Read-only.
1705 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1706 #  Description: Comments about the changelist.  Required.
1707 #  Jobs:        What opened jobs are to be closed by this changelist.
1708 #               You may delete jobs from this list.  (New changelists only.)
1709 #  Files:       What opened files from the default changelist are to be added
1710 #               to this changelist.  You may delete files from this list.
1711 #               (New changelists only.)
1712 """
1713         files_list = []
1714         inFilesSection = False
1715         change_entry = None
1716         args = ['change', '-o']
1717         if changelist:
1718             args.append(str(changelist))
1719         for entry in p4CmdList(args):
1720             if 'code' not in entry:
1721                 continue
1722             if entry['code'] == 'stat':
1723                 change_entry = entry
1724                 break
1725         if not change_entry:
1726             die('Failed to decode output of p4 change -o')
1727         for key, value in change_entry.iteritems():
1728             if key.startswith('File'):
1729                 if 'depot-paths' in settings:
1730                     if not [p for p in settings['depot-paths']
1731                             if p4PathStartsWith(value, p)]:
1732                         continue
1733                 else:
1734                     if not p4PathStartsWith(value, self.depotPath):
1735                         continue
1736                 files_list.append(value)
1737                 continue
1738         # Output in the order expected by prepareLogMessage
1739         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1740             if key not in change_entry:
1741                 continue
1742             template += '\n'
1743             template += key + ':'
1744             if key == 'Description':
1745                 template += '\n'
1746             for field_line in change_entry[key].splitlines():
1747                 template += '\t'+field_line+'\n'
1748         if len(files_list) > 0:
1749             template += '\n'
1750             template += 'Files:\n'
1751         for path in files_list:
1752             template += '\t'+path+'\n'
1753         return template
1754
1755     def edit_template(self, template_file):
1756         """Invoke the editor to let the user change the submission
1757            message.  Return true if okay to continue with the submit."""
1758
1759         # if configured to skip the editing part, just submit
1760         if gitConfigBool("git-p4.skipSubmitEdit"):
1761             return True
1762
1763         # look at the modification time, to check later if the user saved
1764         # the file
1765         mtime = os.stat(template_file).st_mtime
1766
1767         # invoke the editor
1768         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1769             editor = os.environ.get("P4EDITOR")
1770         else:
1771             editor = read_pipe("git var GIT_EDITOR").strip()
1772         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1773
1774         # If the file was not saved, prompt to see if this patch should
1775         # be skipped.  But skip this verification step if configured so.
1776         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1777             return True
1778
1779         # modification time updated means user saved the file
1780         if os.stat(template_file).st_mtime > mtime:
1781             return True
1782
1783         while True:
1784             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1785             if response == 'y':
1786                 return True
1787             if response == 'n':
1788                 return False
1789
1790     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1791         # diff
1792         if "P4DIFF" in os.environ:
1793             del(os.environ["P4DIFF"])
1794         diff = ""
1795         for editedFile in editedFiles:
1796             diff += p4_read_pipe(['diff', '-du',
1797                                   wildcard_encode(editedFile)])
1798
1799         # new file diff
1800         newdiff = ""
1801         for newFile in filesToAdd:
1802             newdiff += "==== new file ====\n"
1803             newdiff += "--- /dev/null\n"
1804             newdiff += "+++ %s\n" % newFile
1805
1806             is_link = os.path.islink(newFile)
1807             expect_link = newFile in symlinks
1808
1809             if is_link and expect_link:
1810                 newdiff += "+%s\n" % os.readlink(newFile)
1811             else:
1812                 f = open(newFile, "r")
1813                 for line in f.readlines():
1814                     newdiff += "+" + line
1815                 f.close()
1816
1817         return (diff + newdiff).replace('\r\n', '\n')
1818
1819     def applyCommit(self, id):
1820         """Apply one commit, return True if it succeeded."""
1821
1822         print("Applying", read_pipe(["git", "show", "-s",
1823                                      "--format=format:%h %s", id]))
1824
1825         (p4User, gitEmail) = self.p4UserForCommit(id)
1826
1827         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1828         filesToAdd = set()
1829         filesToChangeType = set()
1830         filesToDelete = set()
1831         editedFiles = set()
1832         pureRenameCopy = set()
1833         symlinks = set()
1834         filesToChangeExecBit = {}
1835         all_files = list()
1836
1837         for line in diff:
1838             diff = parseDiffTreeEntry(line)
1839             modifier = diff['status']
1840             path = diff['src']
1841             all_files.append(path)
1842
1843             if modifier == "M":
1844                 p4_edit(path)
1845                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1846                     filesToChangeExecBit[path] = diff['dst_mode']
1847                 editedFiles.add(path)
1848             elif modifier == "A":
1849                 filesToAdd.add(path)
1850                 filesToChangeExecBit[path] = diff['dst_mode']
1851                 if path in filesToDelete:
1852                     filesToDelete.remove(path)
1853
1854                 dst_mode = int(diff['dst_mode'], 8)
1855                 if dst_mode == 0o120000:
1856                     symlinks.add(path)
1857
1858             elif modifier == "D":
1859                 filesToDelete.add(path)
1860                 if path in filesToAdd:
1861                     filesToAdd.remove(path)
1862             elif modifier == "C":
1863                 src, dest = diff['src'], diff['dst']
1864                 all_files.append(dest)
1865                 p4_integrate(src, dest)
1866                 pureRenameCopy.add(dest)
1867                 if diff['src_sha1'] != diff['dst_sha1']:
1868                     p4_edit(dest)
1869                     pureRenameCopy.discard(dest)
1870                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1871                     p4_edit(dest)
1872                     pureRenameCopy.discard(dest)
1873                     filesToChangeExecBit[dest] = diff['dst_mode']
1874                 if self.isWindows:
1875                     # turn off read-only attribute
1876                     os.chmod(dest, stat.S_IWRITE)
1877                 os.unlink(dest)
1878                 editedFiles.add(dest)
1879             elif modifier == "R":
1880                 src, dest = diff['src'], diff['dst']
1881                 all_files.append(dest)
1882                 if self.p4HasMoveCommand:
1883                     p4_edit(src)        # src must be open before move
1884                     p4_move(src, dest)  # opens for (move/delete, move/add)
1885                 else:
1886                     p4_integrate(src, dest)
1887                     if diff['src_sha1'] != diff['dst_sha1']:
1888                         p4_edit(dest)
1889                     else:
1890                         pureRenameCopy.add(dest)
1891                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1892                     if not self.p4HasMoveCommand:
1893                         p4_edit(dest)   # with move: already open, writable
1894                     filesToChangeExecBit[dest] = diff['dst_mode']
1895                 if not self.p4HasMoveCommand:
1896                     if self.isWindows:
1897                         os.chmod(dest, stat.S_IWRITE)
1898                     os.unlink(dest)
1899                     filesToDelete.add(src)
1900                 editedFiles.add(dest)
1901             elif modifier == "T":
1902                 filesToChangeType.add(path)
1903             else:
1904                 die("unknown modifier %s for %s" % (modifier, path))
1905
1906         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1907         patchcmd = diffcmd + " | git apply "
1908         tryPatchCmd = patchcmd + "--check -"
1909         applyPatchCmd = patchcmd + "--check --apply -"
1910         patch_succeeded = True
1911
1912         if os.system(tryPatchCmd) != 0:
1913             fixed_rcs_keywords = False
1914             patch_succeeded = False
1915             print("Unfortunately applying the change failed!")
1916
1917             # Patch failed, maybe it's just RCS keyword woes. Look through
1918             # the patch to see if that's possible.
1919             if gitConfigBool("git-p4.attemptRCSCleanup"):
1920                 file = None
1921                 pattern = None
1922                 kwfiles = {}
1923                 for file in editedFiles | filesToDelete:
1924                     # did this file's delta contain RCS keywords?
1925                     pattern = p4_keywords_regexp_for_file(file)
1926
1927                     if pattern:
1928                         # this file is a possibility...look for RCS keywords.
1929                         regexp = re.compile(pattern, re.VERBOSE)
1930                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1931                             if regexp.search(line):
1932                                 if verbose:
1933                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1934                                 kwfiles[file] = pattern
1935                                 break
1936
1937                 for file in kwfiles:
1938                     if verbose:
1939                         print("zapping %s with %s" % (line,pattern))
1940                     # File is being deleted, so not open in p4.  Must
1941                     # disable the read-only bit on windows.
1942                     if self.isWindows and file not in editedFiles:
1943                         os.chmod(file, stat.S_IWRITE)
1944                     self.patchRCSKeywords(file, kwfiles[file])
1945                     fixed_rcs_keywords = True
1946
1947             if fixed_rcs_keywords:
1948                 print("Retrying the patch with RCS keywords cleaned up")
1949                 if os.system(tryPatchCmd) == 0:
1950                     patch_succeeded = True
1951
1952         if not patch_succeeded:
1953             for f in editedFiles:
1954                 p4_revert(f)
1955             return False
1956
1957         #
1958         # Apply the patch for real, and do add/delete/+x handling.
1959         #
1960         system(applyPatchCmd)
1961
1962         for f in filesToChangeType:
1963             p4_edit(f, "-t", "auto")
1964         for f in filesToAdd:
1965             p4_add(f)
1966         for f in filesToDelete:
1967             p4_revert(f)
1968             p4_delete(f)
1969
1970         # Set/clear executable bits
1971         for f in filesToChangeExecBit.keys():
1972             mode = filesToChangeExecBit[f]
1973             setP4ExecBit(f, mode)
1974
1975         update_shelve = 0
1976         if len(self.update_shelve) > 0:
1977             update_shelve = self.update_shelve.pop(0)
1978             p4_reopen_in_change(update_shelve, all_files)
1979
1980         #
1981         # Build p4 change description, starting with the contents
1982         # of the git commit message.
1983         #
1984         logMessage = extractLogMessageFromGitCommit(id)
1985         logMessage = logMessage.strip()
1986         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1987
1988         template = self.prepareSubmitTemplate(update_shelve)
1989         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1990
1991         if self.preserveUser:
1992            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1993
1994         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1995             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1996             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1997             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1998
1999         separatorLine = "######## everything below this line is just the diff #######\n"
2000         if not self.prepare_p4_only:
2001             submitTemplate += separatorLine
2002             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2003
2004         (handle, fileName) = tempfile.mkstemp()
2005         tmpFile = os.fdopen(handle, "w+b")
2006         if self.isWindows:
2007             submitTemplate = submitTemplate.replace("\n", "\r\n")
2008         tmpFile.write(submitTemplate)
2009         tmpFile.close()
2010
2011         if self.prepare_p4_only:
2012             #
2013             # Leave the p4 tree prepared, and the submit template around
2014             # and let the user decide what to do next
2015             #
2016             print()
2017             print("P4 workspace prepared for submission.")
2018             print("To submit or revert, go to client workspace")
2019             print("  " + self.clientPath)
2020             print()
2021             print("To submit, use \"p4 submit\" to write a new description,")
2022             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2023                   " \"git p4\"." % fileName)
2024             print("You can delete the file \"%s\" when finished." % fileName)
2025
2026             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2027                 print("To preserve change ownership by user %s, you must\n" \
2028                       "do \"p4 change -f <change>\" after submitting and\n" \
2029                       "edit the User field.")
2030             if pureRenameCopy:
2031                 print("After submitting, renamed files must be re-synced.")
2032                 print("Invoke \"p4 sync -f\" on each of these files:")
2033                 for f in pureRenameCopy:
2034                     print("  " + f)
2035
2036             print()
2037             print("To revert the changes, use \"p4 revert ...\", and delete")
2038             print("the submit template file \"%s\"" % fileName)
2039             if filesToAdd:
2040                 print("Since the commit adds new files, they must be deleted:")
2041                 for f in filesToAdd:
2042                     print("  " + f)
2043             print()
2044             return True
2045
2046         #
2047         # Let the user edit the change description, then submit it.
2048         #
2049         submitted = False
2050
2051         try:
2052             if self.edit_template(fileName):
2053                 # read the edited message and submit
2054                 tmpFile = open(fileName, "rb")
2055                 message = tmpFile.read()
2056                 tmpFile.close()
2057                 if self.isWindows:
2058                     message = message.replace("\r\n", "\n")
2059                 submitTemplate = message[:message.index(separatorLine)]
2060
2061                 if update_shelve:
2062                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2063                 elif self.shelve:
2064                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2065                 else:
2066                     p4_write_pipe(['submit', '-i'], submitTemplate)
2067                     # The rename/copy happened by applying a patch that created a
2068                     # new file.  This leaves it writable, which confuses p4.
2069                     for f in pureRenameCopy:
2070                         p4_sync(f, "-f")
2071
2072                 if self.preserveUser:
2073                     if p4User:
2074                         # Get last changelist number. Cannot easily get it from
2075                         # the submit command output as the output is
2076                         # unmarshalled.
2077                         changelist = self.lastP4Changelist()
2078                         self.modifyChangelistUser(changelist, p4User)
2079
2080                 submitted = True
2081
2082         finally:
2083             # skip this patch
2084             if not submitted or self.shelve:
2085                 if self.shelve:
2086                     print ("Reverting shelved files.")
2087                 else:
2088                     print ("Submission cancelled, undoing p4 changes.")
2089                 for f in editedFiles | filesToDelete:
2090                     p4_revert(f)
2091                 for f in filesToAdd:
2092                     p4_revert(f)
2093                     os.remove(f)
2094
2095         os.remove(fileName)
2096         return submitted
2097
2098     # Export git tags as p4 labels. Create a p4 label and then tag
2099     # with that.
2100     def exportGitTags(self, gitTags):
2101         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2102         if len(validLabelRegexp) == 0:
2103             validLabelRegexp = defaultLabelRegexp
2104         m = re.compile(validLabelRegexp)
2105
2106         for name in gitTags:
2107
2108             if not m.match(name):
2109                 if verbose:
2110                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2111                 continue
2112
2113             # Get the p4 commit this corresponds to
2114             logMessage = extractLogMessageFromGitCommit(name)
2115             values = extractSettingsGitLog(logMessage)
2116
2117             if 'change' not in values:
2118                 # a tag pointing to something not sent to p4; ignore
2119                 if verbose:
2120                     print("git tag %s does not give a p4 commit" % name)
2121                 continue
2122             else:
2123                 changelist = values['change']
2124
2125             # Get the tag details.
2126             inHeader = True
2127             isAnnotated = False
2128             body = []
2129             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2130                 l = l.strip()
2131                 if inHeader:
2132                     if re.match(r'tag\s+', l):
2133                         isAnnotated = True
2134                     elif re.match(r'\s*$', l):
2135                         inHeader = False
2136                         continue
2137                 else:
2138                     body.append(l)
2139
2140             if not isAnnotated:
2141                 body = ["lightweight tag imported by git p4\n"]
2142
2143             # Create the label - use the same view as the client spec we are using
2144             clientSpec = getClientSpec()
2145
2146             labelTemplate  = "Label: %s\n" % name
2147             labelTemplate += "Description:\n"
2148             for b in body:
2149                 labelTemplate += "\t" + b + "\n"
2150             labelTemplate += "View:\n"
2151             for depot_side in clientSpec.mappings:
2152                 labelTemplate += "\t%s\n" % depot_side
2153
2154             if self.dry_run:
2155                 print("Would create p4 label %s for tag" % name)
2156             elif self.prepare_p4_only:
2157                 print("Not creating p4 label %s for tag due to option" \
2158                       " --prepare-p4-only" % name)
2159             else:
2160                 p4_write_pipe(["label", "-i"], labelTemplate)
2161
2162                 # Use the label
2163                 p4_system(["tag", "-l", name] +
2164                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2165
2166                 if verbose:
2167                     print("created p4 label for tag %s" % name)
2168
2169     def run(self, args):
2170         if len(args) == 0:
2171             self.master = currentGitBranch()
2172         elif len(args) == 1:
2173             self.master = args[0]
2174             if not branchExists(self.master):
2175                 die("Branch %s does not exist" % self.master)
2176         else:
2177             return False
2178
2179         for i in self.update_shelve:
2180             if i <= 0:
2181                 sys.exit("invalid changelist %d" % i)
2182
2183         if self.master:
2184             allowSubmit = gitConfig("git-p4.allowSubmit")
2185             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2186                 die("%s is not in git-p4.allowSubmit" % self.master)
2187
2188         [upstream, settings] = findUpstreamBranchPoint()
2189         self.depotPath = settings['depot-paths'][0]
2190         if len(self.origin) == 0:
2191             self.origin = upstream
2192
2193         if len(self.update_shelve) > 0:
2194             self.shelve = True
2195
2196         if self.preserveUser:
2197             if not self.canChangeChangelists():
2198                 die("Cannot preserve user names without p4 super-user or admin permissions")
2199
2200         # if not set from the command line, try the config file
2201         if self.conflict_behavior is None:
2202             val = gitConfig("git-p4.conflict")
2203             if val:
2204                 if val not in self.conflict_behavior_choices:
2205                     die("Invalid value '%s' for config git-p4.conflict" % val)
2206             else:
2207                 val = "ask"
2208             self.conflict_behavior = val
2209
2210         if self.verbose:
2211             print("Origin branch is " + self.origin)
2212
2213         if len(self.depotPath) == 0:
2214             print("Internal error: cannot locate perforce depot path from existing branches")
2215             sys.exit(128)
2216
2217         self.useClientSpec = False
2218         if gitConfigBool("git-p4.useclientspec"):
2219             self.useClientSpec = True
2220         if self.useClientSpec:
2221             self.clientSpecDirs = getClientSpec()
2222
2223         # Check for the existence of P4 branches
2224         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2225
2226         if self.useClientSpec and not branchesDetected:
2227             # all files are relative to the client spec
2228             self.clientPath = getClientRoot()
2229         else:
2230             self.clientPath = p4Where(self.depotPath)
2231
2232         if self.clientPath == "":
2233             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2234
2235         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2236         self.oldWorkingDirectory = os.getcwd()
2237
2238         # ensure the clientPath exists
2239         new_client_dir = False
2240         if not os.path.exists(self.clientPath):
2241             new_client_dir = True
2242             os.makedirs(self.clientPath)
2243
2244         chdir(self.clientPath, is_client_path=True)
2245         if self.dry_run:
2246             print("Would synchronize p4 checkout in %s" % self.clientPath)
2247         else:
2248             print("Synchronizing p4 checkout...")
2249             if new_client_dir:
2250                 # old one was destroyed, and maybe nobody told p4
2251                 p4_sync("...", "-f")
2252             else:
2253                 p4_sync("...")
2254         self.check()
2255
2256         commits = []
2257         if self.master:
2258             committish = self.master
2259         else:
2260             committish = 'HEAD'
2261
2262         if self.commit != "":
2263             if self.commit.find("..") != -1:
2264                 limits_ish = self.commit.split("..")
2265                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2266                     commits.append(line.strip())
2267                 commits.reverse()
2268             else:
2269                 commits.append(self.commit)
2270         else:
2271             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2272                 commits.append(line.strip())
2273             commits.reverse()
2274
2275         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2276             self.checkAuthorship = False
2277         else:
2278             self.checkAuthorship = True
2279
2280         if self.preserveUser:
2281             self.checkValidP4Users(commits)
2282
2283         #
2284         # Build up a set of options to be passed to diff when
2285         # submitting each commit to p4.
2286         #
2287         if self.detectRenames:
2288             # command-line -M arg
2289             self.diffOpts = "-M"
2290         else:
2291             # If not explicitly set check the config variable
2292             detectRenames = gitConfig("git-p4.detectRenames")
2293
2294             if detectRenames.lower() == "false" or detectRenames == "":
2295                 self.diffOpts = ""
2296             elif detectRenames.lower() == "true":
2297                 self.diffOpts = "-M"
2298             else:
2299                 self.diffOpts = "-M%s" % detectRenames
2300
2301         # no command-line arg for -C or --find-copies-harder, just
2302         # config variables
2303         detectCopies = gitConfig("git-p4.detectCopies")
2304         if detectCopies.lower() == "false" or detectCopies == "":
2305             pass
2306         elif detectCopies.lower() == "true":
2307             self.diffOpts += " -C"
2308         else:
2309             self.diffOpts += " -C%s" % detectCopies
2310
2311         if gitConfigBool("git-p4.detectCopiesHarder"):
2312             self.diffOpts += " --find-copies-harder"
2313
2314         num_shelves = len(self.update_shelve)
2315         if num_shelves > 0 and num_shelves != len(commits):
2316             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2317                      (len(commits), num_shelves))
2318
2319         hooks_path = gitConfig("core.hooksPath")
2320         if len(hooks_path) <= 0:
2321             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2322
2323         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2324         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2325             sys.exit(1)
2326
2327         #
2328         # Apply the commits, one at a time.  On failure, ask if should
2329         # continue to try the rest of the patches, or quit.
2330         #
2331         if self.dry_run:
2332             print("Would apply")
2333         applied = []
2334         last = len(commits) - 1
2335         for i, commit in enumerate(commits):
2336             if self.dry_run:
2337                 print(" ", read_pipe(["git", "show", "-s",
2338                                       "--format=format:%h %s", commit]))
2339                 ok = True
2340             else:
2341                 ok = self.applyCommit(commit)
2342             if ok:
2343                 applied.append(commit)
2344             else:
2345                 if self.prepare_p4_only and i < last:
2346                     print("Processing only the first commit due to option" \
2347                           " --prepare-p4-only")
2348                     break
2349                 if i < last:
2350                     quit = False
2351                     while True:
2352                         # prompt for what to do, or use the option/variable
2353                         if self.conflict_behavior == "ask":
2354                             print("What do you want to do?")
2355                             response = raw_input("[s]kip this commit but apply"
2356                                                  " the rest, or [q]uit? ")
2357                             if not response:
2358                                 continue
2359                         elif self.conflict_behavior == "skip":
2360                             response = "s"
2361                         elif self.conflict_behavior == "quit":
2362                             response = "q"
2363                         else:
2364                             die("Unknown conflict_behavior '%s'" %
2365                                 self.conflict_behavior)
2366
2367                         if response[0] == "s":
2368                             print("Skipping this commit, but applying the rest")
2369                             break
2370                         if response[0] == "q":
2371                             print("Quitting")
2372                             quit = True
2373                             break
2374                     if quit:
2375                         break
2376
2377         chdir(self.oldWorkingDirectory)
2378         shelved_applied = "shelved" if self.shelve else "applied"
2379         if self.dry_run:
2380             pass
2381         elif self.prepare_p4_only:
2382             pass
2383         elif len(commits) == len(applied):
2384             print("All commits {0}!".format(shelved_applied))
2385
2386             sync = P4Sync()
2387             if self.branch:
2388                 sync.branch = self.branch
2389             if self.disable_p4sync:
2390                 sync.sync_origin_only()
2391             else:
2392                 sync.run([])
2393
2394                 if not self.disable_rebase:
2395                     rebase = P4Rebase()
2396                     rebase.rebase()
2397
2398         else:
2399             if len(applied) == 0:
2400                 print("No commits {0}.".format(shelved_applied))
2401             else:
2402                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2403                 for c in commits:
2404                     if c in applied:
2405                         star = "*"
2406                     else:
2407                         star = " "
2408                     print(star, read_pipe(["git", "show", "-s",
2409                                            "--format=format:%h %s",  c]))
2410                 print("You will have to do 'git p4 sync' and rebase.")
2411
2412         if gitConfigBool("git-p4.exportLabels"):
2413             self.exportLabels = True
2414
2415         if self.exportLabels:
2416             p4Labels = getP4Labels(self.depotPath)
2417             gitTags = getGitTags()
2418
2419             missingGitTags = gitTags - p4Labels
2420             self.exportGitTags(missingGitTags)
2421
2422         # exit with error unless everything applied perfectly
2423         if len(commits) != len(applied):
2424                 sys.exit(1)
2425
2426         return True
2427
2428 class View(object):
2429     """Represent a p4 view ("p4 help views"), and map files in a
2430        repo according to the view."""
2431
2432     def __init__(self, client_name):
2433         self.mappings = []
2434         self.client_prefix = "//%s/" % client_name
2435         # cache results of "p4 where" to lookup client file locations
2436         self.client_spec_path_cache = {}
2437
2438     def append(self, view_line):
2439         """Parse a view line, splitting it into depot and client
2440            sides.  Append to self.mappings, preserving order.  This
2441            is only needed for tag creation."""
2442
2443         # Split the view line into exactly two words.  P4 enforces
2444         # structure on these lines that simplifies this quite a bit.
2445         #
2446         # Either or both words may be double-quoted.
2447         # Single quotes do not matter.
2448         # Double-quote marks cannot occur inside the words.
2449         # A + or - prefix is also inside the quotes.
2450         # There are no quotes unless they contain a space.
2451         # The line is already white-space stripped.
2452         # The two words are separated by a single space.
2453         #
2454         if view_line[0] == '"':
2455             # First word is double quoted.  Find its end.
2456             close_quote_index = view_line.find('"', 1)
2457             if close_quote_index <= 0:
2458                 die("No first-word closing quote found: %s" % view_line)
2459             depot_side = view_line[1:close_quote_index]
2460             # skip closing quote and space
2461             rhs_index = close_quote_index + 1 + 1
2462         else:
2463             space_index = view_line.find(" ")
2464             if space_index <= 0:
2465                 die("No word-splitting space found: %s" % view_line)
2466             depot_side = view_line[0:space_index]
2467             rhs_index = space_index + 1
2468
2469         # prefix + means overlay on previous mapping
2470         if depot_side.startswith("+"):
2471             depot_side = depot_side[1:]
2472
2473         # prefix - means exclude this path, leave out of mappings
2474         exclude = False
2475         if depot_side.startswith("-"):
2476             exclude = True
2477             depot_side = depot_side[1:]
2478
2479         if not exclude:
2480             self.mappings.append(depot_side)
2481
2482     def convert_client_path(self, clientFile):
2483         # chop off //client/ part to make it relative
2484         if not clientFile.startswith(self.client_prefix):
2485             die("No prefix '%s' on clientFile '%s'" %
2486                 (self.client_prefix, clientFile))
2487         return clientFile[len(self.client_prefix):]
2488
2489     def update_client_spec_path_cache(self, files):
2490         """ Caching file paths by "p4 where" batch query """
2491
2492         # List depot file paths exclude that already cached
2493         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2494
2495         if len(fileArgs) == 0:
2496             return  # All files in cache
2497
2498         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2499         for res in where_result:
2500             if "code" in res and res["code"] == "error":
2501                 # assume error is "... file(s) not in client view"
2502                 continue
2503             if "clientFile" not in res:
2504                 die("No clientFile in 'p4 where' output")
2505             if "unmap" in res:
2506                 # it will list all of them, but only one not unmap-ped
2507                 continue
2508             if gitConfigBool("core.ignorecase"):
2509                 res['depotFile'] = res['depotFile'].lower()
2510             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2511
2512         # not found files or unmap files set to ""
2513         for depotFile in fileArgs:
2514             if gitConfigBool("core.ignorecase"):
2515                 depotFile = depotFile.lower()
2516             if depotFile not in self.client_spec_path_cache:
2517                 self.client_spec_path_cache[depotFile] = ""
2518
2519     def map_in_client(self, depot_path):
2520         """Return the relative location in the client where this
2521            depot file should live.  Returns "" if the file should
2522            not be mapped in the client."""
2523
2524         if gitConfigBool("core.ignorecase"):
2525             depot_path = depot_path.lower()
2526
2527         if depot_path in self.client_spec_path_cache:
2528             return self.client_spec_path_cache[depot_path]
2529
2530         die( "Error: %s is not found in client spec path" % depot_path )
2531         return ""
2532
2533 class P4Sync(Command, P4UserMap):
2534
2535     def __init__(self):
2536         Command.__init__(self)
2537         P4UserMap.__init__(self)
2538         self.options = [
2539                 optparse.make_option("--branch", dest="branch"),
2540                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2541                 optparse.make_option("--changesfile", dest="changesFile"),
2542                 optparse.make_option("--silent", dest="silent", action="store_true"),
2543                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2544                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2545                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2546                                      help="Import into refs/heads/ , not refs/remotes"),
2547                 optparse.make_option("--max-changes", dest="maxChanges",
2548                                      help="Maximum number of changes to import"),
2549                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2550                                      help="Internal block size to use when iteratively calling p4 changes"),
2551                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2552                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2553                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2554                                      help="Only sync files that are included in the Perforce Client Spec"),
2555                 optparse.make_option("-/", dest="cloneExclude",
2556                                      action="append", type="string",
2557                                      help="exclude depot path"),
2558         ]
2559         self.description = """Imports from Perforce into a git repository.\n
2560     example:
2561     //depot/my/project/ -- to import the current head
2562     //depot/my/project/@all -- to import everything
2563     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2564
2565     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2566
2567         self.usage += " //depot/path[@revRange]"
2568         self.silent = False
2569         self.createdBranches = set()
2570         self.committedChanges = set()
2571         self.branch = ""
2572         self.detectBranches = False
2573         self.detectLabels = False
2574         self.importLabels = False
2575         self.changesFile = ""
2576         self.syncWithOrigin = True
2577         self.importIntoRemotes = True
2578         self.maxChanges = ""
2579         self.changes_block_size = None
2580         self.keepRepoPath = False
2581         self.depotPaths = None
2582         self.p4BranchesInGit = []
2583         self.cloneExclude = []
2584         self.useClientSpec = False
2585         self.useClientSpec_from_options = False
2586         self.clientSpecDirs = None
2587         self.tempBranches = []
2588         self.tempBranchLocation = "refs/git-p4-tmp"
2589         self.largeFileSystem = None
2590         self.suppress_meta_comment = False
2591
2592         if gitConfig('git-p4.largeFileSystem'):
2593             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2594             self.largeFileSystem = largeFileSystemConstructor(
2595                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2596             )
2597
2598         if gitConfig("git-p4.syncFromOrigin") == "false":
2599             self.syncWithOrigin = False
2600
2601         self.depotPaths = []
2602         self.changeRange = ""
2603         self.previousDepotPaths = []
2604         self.hasOrigin = False
2605
2606         # map from branch depot path to parent branch
2607         self.knownBranches = {}
2608         self.initialParents = {}
2609
2610         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2611         self.labels = {}
2612
2613     # Force a checkpoint in fast-import and wait for it to finish
2614     def checkpoint(self):
2615         self.gitStream.write("checkpoint\n\n")
2616         self.gitStream.write("progress checkpoint\n\n")
2617         out = self.gitOutput.readline()
2618         if self.verbose:
2619             print("checkpoint finished: " + out)
2620
2621     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2622         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2623                              for path in self.cloneExclude]
2624         files = []
2625         fnum = 0
2626         while "depotFile%s" % fnum in commit:
2627             path =  commit["depotFile%s" % fnum]
2628
2629             if [p for p in self.cloneExclude
2630                 if p4PathStartsWith(path, p)]:
2631                 found = False
2632             else:
2633                 found = [p for p in self.depotPaths
2634                          if p4PathStartsWith(path, p)]
2635             if not found:
2636                 fnum = fnum + 1
2637                 continue
2638
2639             file = {}
2640             file["path"] = path
2641             file["rev"] = commit["rev%s" % fnum]
2642             file["action"] = commit["action%s" % fnum]
2643             file["type"] = commit["type%s" % fnum]
2644             if shelved:
2645                 file["shelved_cl"] = int(shelved_cl)
2646             files.append(file)
2647             fnum = fnum + 1
2648         return files
2649
2650     def extractJobsFromCommit(self, commit):
2651         jobs = []
2652         jnum = 0
2653         while "job%s" % jnum in commit:
2654             job = commit["job%s" % jnum]
2655             jobs.append(job)
2656             jnum = jnum + 1
2657         return jobs
2658
2659     def stripRepoPath(self, path, prefixes):
2660         """When streaming files, this is called to map a p4 depot path
2661            to where it should go in git.  The prefixes are either
2662            self.depotPaths, or self.branchPrefixes in the case of
2663            branch detection."""
2664
2665         if self.useClientSpec:
2666             # branch detection moves files up a level (the branch name)
2667             # from what client spec interpretation gives
2668             path = self.clientSpecDirs.map_in_client(path)
2669             if self.detectBranches:
2670                 for b in self.knownBranches:
2671                     if path.startswith(b + "/"):
2672                         path = path[len(b)+1:]
2673
2674         elif self.keepRepoPath:
2675             # Preserve everything in relative path name except leading
2676             # //depot/; just look at first prefix as they all should
2677             # be in the same depot.
2678             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2679             if p4PathStartsWith(path, depot):
2680                 path = path[len(depot):]
2681
2682         else:
2683             for p in prefixes:
2684                 if p4PathStartsWith(path, p):
2685                     path = path[len(p):]
2686                     break
2687
2688         path = wildcard_decode(path)
2689         return path
2690
2691     def splitFilesIntoBranches(self, commit):
2692         """Look at each depotFile in the commit to figure out to what
2693            branch it belongs."""
2694
2695         if self.clientSpecDirs:
2696             files = self.extractFilesFromCommit(commit)
2697             self.clientSpecDirs.update_client_spec_path_cache(files)
2698
2699         branches = {}
2700         fnum = 0
2701         while "depotFile%s" % fnum in commit:
2702             path =  commit["depotFile%s" % fnum]
2703             found = [p for p in self.depotPaths
2704                      if p4PathStartsWith(path, p)]
2705             if not found:
2706                 fnum = fnum + 1
2707                 continue
2708
2709             file = {}
2710             file["path"] = path
2711             file["rev"] = commit["rev%s" % fnum]
2712             file["action"] = commit["action%s" % fnum]
2713             file["type"] = commit["type%s" % fnum]
2714             fnum = fnum + 1
2715
2716             # start with the full relative path where this file would
2717             # go in a p4 client
2718             if self.useClientSpec:
2719                 relPath = self.clientSpecDirs.map_in_client(path)
2720             else:
2721                 relPath = self.stripRepoPath(path, self.depotPaths)
2722
2723             for branch in self.knownBranches.keys():
2724                 # add a trailing slash so that a commit into qt/4.2foo
2725                 # doesn't end up in qt/4.2, e.g.
2726                 if relPath.startswith(branch + "/"):
2727                     if branch not in branches:
2728                         branches[branch] = []
2729                     branches[branch].append(file)
2730                     break
2731
2732         return branches
2733
2734     def writeToGitStream(self, gitMode, relPath, contents):
2735         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2736         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2737         for d in contents:
2738             self.gitStream.write(d)
2739         self.gitStream.write('\n')
2740
2741     def encodeWithUTF8(self, path):
2742         try:
2743             path.decode('ascii')
2744         except:
2745             encoding = 'utf8'
2746             if gitConfig('git-p4.pathEncoding'):
2747                 encoding = gitConfig('git-p4.pathEncoding')
2748             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2749             if self.verbose:
2750                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2751         return path
2752
2753     # output one file from the P4 stream
2754     # - helper for streamP4Files
2755
2756     def streamOneP4File(self, file, contents):
2757         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2758         relPath = self.encodeWithUTF8(relPath)
2759         if verbose:
2760             if 'fileSize' in self.stream_file:
2761                 size = int(self.stream_file['fileSize'])
2762             else:
2763                 size = 0 # deleted files don't get a fileSize apparently
2764             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2765             sys.stdout.flush()
2766
2767         (type_base, type_mods) = split_p4_type(file["type"])
2768
2769         git_mode = "100644"
2770         if "x" in type_mods:
2771             git_mode = "100755"
2772         if type_base == "symlink":
2773             git_mode = "120000"
2774             # p4 print on a symlink sometimes contains "target\n";
2775             # if it does, remove the newline
2776             data = ''.join(contents)
2777             if not data:
2778                 # Some version of p4 allowed creating a symlink that pointed
2779                 # to nothing.  This causes p4 errors when checking out such
2780                 # a change, and errors here too.  Work around it by ignoring
2781                 # the bad symlink; hopefully a future change fixes it.
2782                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2783                 return
2784             elif data[-1] == '\n':
2785                 contents = [data[:-1]]
2786             else:
2787                 contents = [data]
2788
2789         if type_base == "utf16":
2790             # p4 delivers different text in the python output to -G
2791             # than it does when using "print -o", or normal p4 client
2792             # operations.  utf16 is converted to ascii or utf8, perhaps.
2793             # But ascii text saved as -t utf16 is completely mangled.
2794             # Invoke print -o to get the real contents.
2795             #
2796             # On windows, the newlines will always be mangled by print, so put
2797             # them back too.  This is not needed to the cygwin windows version,
2798             # just the native "NT" type.
2799             #
2800             try:
2801                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2802             except Exception as e:
2803                 if 'Translation of file content failed' in str(e):
2804                     type_base = 'binary'
2805                 else:
2806                     raise e
2807             else:
2808                 if p4_version_string().find('/NT') >= 0:
2809                     text = text.replace('\r\n', '\n')
2810                 contents = [ text ]
2811
2812         if type_base == "apple":
2813             # Apple filetype files will be streamed as a concatenation of
2814             # its appledouble header and the contents.  This is useless
2815             # on both macs and non-macs.  If using "print -q -o xx", it
2816             # will create "xx" with the data, and "%xx" with the header.
2817             # This is also not very useful.
2818             #
2819             # Ideally, someday, this script can learn how to generate
2820             # appledouble files directly and import those to git, but
2821             # non-mac machines can never find a use for apple filetype.
2822             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2823             return
2824
2825         # Note that we do not try to de-mangle keywords on utf16 files,
2826         # even though in theory somebody may want that.
2827         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2828         if pattern:
2829             regexp = re.compile(pattern, re.VERBOSE)
2830             text = ''.join(contents)
2831             text = regexp.sub(r'$\1$', text)
2832             contents = [ text ]
2833
2834         if self.largeFileSystem:
2835             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2836
2837         self.writeToGitStream(git_mode, relPath, contents)
2838
2839     def streamOneP4Deletion(self, file):
2840         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2841         relPath = self.encodeWithUTF8(relPath)
2842         if verbose:
2843             sys.stdout.write("delete %s\n" % relPath)
2844             sys.stdout.flush()
2845         self.gitStream.write("D %s\n" % relPath)
2846
2847         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2848             self.largeFileSystem.removeLargeFile(relPath)
2849
2850     # handle another chunk of streaming data
2851     def streamP4FilesCb(self, marshalled):
2852
2853         # catch p4 errors and complain
2854         err = None
2855         if "code" in marshalled:
2856             if marshalled["code"] == "error":
2857                 if "data" in marshalled:
2858                     err = marshalled["data"].rstrip()
2859
2860         if not err and 'fileSize' in self.stream_file:
2861             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2862             if required_bytes > 0:
2863                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2864                     os.getcwd(), required_bytes/1024/1024
2865                 )
2866
2867         if err:
2868             f = None
2869             if self.stream_have_file_info:
2870                 if "depotFile" in self.stream_file:
2871                     f = self.stream_file["depotFile"]
2872             # force a failure in fast-import, else an empty
2873             # commit will be made
2874             self.gitStream.write("\n")
2875             self.gitStream.write("die-now\n")
2876             self.gitStream.close()
2877             # ignore errors, but make sure it exits first
2878             self.importProcess.wait()
2879             if f:
2880                 die("Error from p4 print for %s: %s" % (f, err))
2881             else:
2882                 die("Error from p4 print: %s" % err)
2883
2884         if 'depotFile' in marshalled and self.stream_have_file_info:
2885             # start of a new file - output the old one first
2886             self.streamOneP4File(self.stream_file, self.stream_contents)
2887             self.stream_file = {}
2888             self.stream_contents = []
2889             self.stream_have_file_info = False
2890
2891         # pick up the new file information... for the
2892         # 'data' field we need to append to our array
2893         for k in marshalled.keys():
2894             if k == 'data':
2895                 if 'streamContentSize' not in self.stream_file:
2896                     self.stream_file['streamContentSize'] = 0
2897                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2898                 self.stream_contents.append(marshalled['data'])
2899             else:
2900                 self.stream_file[k] = marshalled[k]
2901
2902         if (verbose and
2903             'streamContentSize' in self.stream_file and
2904             'fileSize' in self.stream_file and
2905             'depotFile' in self.stream_file):
2906             size = int(self.stream_file["fileSize"])
2907             if size > 0:
2908                 progress = 100*self.stream_file['streamContentSize']/size
2909                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2910                 sys.stdout.flush()
2911
2912         self.stream_have_file_info = True
2913
2914     # Stream directly from "p4 files" into "git fast-import"
2915     def streamP4Files(self, files):
2916         filesForCommit = []
2917         filesToRead = []
2918         filesToDelete = []
2919
2920         for f in files:
2921             filesForCommit.append(f)
2922             if f['action'] in self.delete_actions:
2923                 filesToDelete.append(f)
2924             else:
2925                 filesToRead.append(f)
2926
2927         # deleted files...
2928         for f in filesToDelete:
2929             self.streamOneP4Deletion(f)
2930
2931         if len(filesToRead) > 0:
2932             self.stream_file = {}
2933             self.stream_contents = []
2934             self.stream_have_file_info = False
2935
2936             # curry self argument
2937             def streamP4FilesCbSelf(entry):
2938                 self.streamP4FilesCb(entry)
2939
2940             fileArgs = []
2941             for f in filesToRead:
2942                 if 'shelved_cl' in f:
2943                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2944                     # the contents
2945                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2946                 else:
2947                     fileArg = '%s#%s' % (f['path'], f['rev'])
2948
2949                 fileArgs.append(fileArg)
2950
2951             p4CmdList(["-x", "-", "print"],
2952                       stdin=fileArgs,
2953                       cb=streamP4FilesCbSelf)
2954
2955             # do the last chunk
2956             if 'depotFile' in self.stream_file:
2957                 self.streamOneP4File(self.stream_file, self.stream_contents)
2958
2959     def make_email(self, userid):
2960         if userid in self.users:
2961             return self.users[userid]
2962         else:
2963             return "%s <a@b>" % userid
2964
2965     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2966         """ Stream a p4 tag.
2967         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2968         """
2969
2970         if verbose:
2971             print("writing tag %s for commit %s" % (labelName, commit))
2972         gitStream.write("tag %s\n" % labelName)
2973         gitStream.write("from %s\n" % commit)
2974
2975         if 'Owner' in labelDetails:
2976             owner = labelDetails["Owner"]
2977         else:
2978             owner = None
2979
2980         # Try to use the owner of the p4 label, or failing that,
2981         # the current p4 user id.
2982         if owner:
2983             email = self.make_email(owner)
2984         else:
2985             email = self.make_email(self.p4UserId())
2986         tagger = "%s %s %s" % (email, epoch, self.tz)
2987
2988         gitStream.write("tagger %s\n" % tagger)
2989
2990         print("labelDetails=",labelDetails)
2991         if 'Description' in labelDetails:
2992             description = labelDetails['Description']
2993         else:
2994             description = 'Label from git p4'
2995
2996         gitStream.write("data %d\n" % len(description))
2997         gitStream.write(description)
2998         gitStream.write("\n")
2999
3000     def inClientSpec(self, path):
3001         if not self.clientSpecDirs:
3002             return True
3003         inClientSpec = self.clientSpecDirs.map_in_client(path)
3004         if not inClientSpec and self.verbose:
3005             print('Ignoring file outside of client spec: {0}'.format(path))
3006         return inClientSpec
3007
3008     def hasBranchPrefix(self, path):
3009         if not self.branchPrefixes:
3010             return True
3011         hasPrefix = [p for p in self.branchPrefixes
3012                         if p4PathStartsWith(path, p)]
3013         if not hasPrefix and self.verbose:
3014             print('Ignoring file outside of prefix: {0}'.format(path))
3015         return hasPrefix
3016
3017     def commit(self, details, files, branch, parent = "", allow_empty=False):
3018         epoch = details["time"]
3019         author = details["user"]
3020         jobs = self.extractJobsFromCommit(details)
3021
3022         if self.verbose:
3023             print('commit into {0}'.format(branch))
3024
3025         if self.clientSpecDirs:
3026             self.clientSpecDirs.update_client_spec_path_cache(files)
3027
3028         files = [f for f in files
3029             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3030
3031         if gitConfigBool('git-p4.keepEmptyCommits'):
3032             allow_empty = True
3033
3034         if not files and not allow_empty:
3035             print('Ignoring revision {0} as it would produce an empty commit.'
3036                 .format(details['change']))
3037             return
3038
3039         self.gitStream.write("commit %s\n" % branch)
3040         self.gitStream.write("mark :%s\n" % details["change"])
3041         self.committedChanges.add(int(details["change"]))
3042         committer = ""
3043         if author not in self.users:
3044             self.getUserMapFromPerforceServer()
3045         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3046
3047         self.gitStream.write("committer %s\n" % committer)
3048
3049         self.gitStream.write("data <<EOT\n")
3050         self.gitStream.write(details["desc"])
3051         if len(jobs) > 0:
3052             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3053
3054         if not self.suppress_meta_comment:
3055             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3056                                 (','.join(self.branchPrefixes), details["change"]))
3057             if len(details['options']) > 0:
3058                 self.gitStream.write(": options = %s" % details['options'])
3059             self.gitStream.write("]\n")
3060
3061         self.gitStream.write("EOT\n\n")
3062
3063         if len(parent) > 0:
3064             if self.verbose:
3065                 print("parent %s" % parent)
3066             self.gitStream.write("from %s\n" % parent)
3067
3068         self.streamP4Files(files)
3069         self.gitStream.write("\n")
3070
3071         change = int(details["change"])
3072
3073         if change in self.labels:
3074             label = self.labels[change]
3075             labelDetails = label[0]
3076             labelRevisions = label[1]
3077             if self.verbose:
3078                 print("Change %s is labelled %s" % (change, labelDetails))
3079
3080             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3081                                                 for p in self.branchPrefixes])
3082
3083             if len(files) == len(labelRevisions):
3084
3085                 cleanedFiles = {}
3086                 for info in files:
3087                     if info["action"] in self.delete_actions:
3088                         continue
3089                     cleanedFiles[info["depotFile"]] = info["rev"]
3090
3091                 if cleanedFiles == labelRevisions:
3092                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3093
3094                 else:
3095                     if not self.silent:
3096                         print("Tag %s does not match with change %s: files do not match."
3097                                % (labelDetails["label"], change))
3098
3099             else:
3100                 if not self.silent:
3101                     print("Tag %s does not match with change %s: file count is different."
3102                            % (labelDetails["label"], change))
3103
3104     # Build a dictionary of changelists and labels, for "detect-labels" option.
3105     def getLabels(self):
3106         self.labels = {}
3107
3108         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3109         if len(l) > 0 and not self.silent:
3110             print("Finding files belonging to labels in %s" % self.depotPaths)
3111
3112         for output in l:
3113             label = output["label"]
3114             revisions = {}
3115             newestChange = 0
3116             if self.verbose:
3117                 print("Querying files for label %s" % label)
3118             for file in p4CmdList(["files"] +
3119                                       ["%s...@%s" % (p, label)
3120                                           for p in self.depotPaths]):
3121                 revisions[file["depotFile"]] = file["rev"]
3122                 change = int(file["change"])
3123                 if change > newestChange:
3124                     newestChange = change
3125
3126             self.labels[newestChange] = [output, revisions]
3127
3128         if self.verbose:
3129             print("Label changes: %s" % self.labels.keys())
3130
3131     # Import p4 labels as git tags. A direct mapping does not
3132     # exist, so assume that if all the files are at the same revision
3133     # then we can use that, or it's something more complicated we should
3134     # just ignore.
3135     def importP4Labels(self, stream, p4Labels):
3136         if verbose:
3137             print("import p4 labels: " + ' '.join(p4Labels))
3138
3139         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3140         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3141         if len(validLabelRegexp) == 0:
3142             validLabelRegexp = defaultLabelRegexp
3143         m = re.compile(validLabelRegexp)
3144
3145         for name in p4Labels:
3146             commitFound = False
3147
3148             if not m.match(name):
3149                 if verbose:
3150                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3151                 continue
3152
3153             if name in ignoredP4Labels:
3154                 continue
3155
3156             labelDetails = p4CmdList(['label', "-o", name])[0]
3157
3158             # get the most recent changelist for each file in this label
3159             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3160                                 for p in self.depotPaths])
3161
3162             if 'change' in change:
3163                 # find the corresponding git commit; take the oldest commit
3164                 changelist = int(change['change'])
3165                 if changelist in self.committedChanges:
3166                     gitCommit = ":%d" % changelist       # use a fast-import mark
3167                     commitFound = True
3168                 else:
3169                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3170                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3171                     if len(gitCommit) == 0:
3172                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3173                     else:
3174                         commitFound = True
3175                         gitCommit = gitCommit.strip()
3176
3177                 if commitFound:
3178                     # Convert from p4 time format
3179                     try:
3180                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3181                     except ValueError:
3182                         print("Could not convert label time %s" % labelDetails['Update'])
3183                         tmwhen = 1
3184
3185                     when = int(time.mktime(tmwhen))
3186                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3187                     if verbose:
3188                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3189             else:
3190                 if verbose:
3191                     print("Label %s has no changelists - possibly deleted?" % name)
3192
3193             if not commitFound:
3194                 # We can't import this label; don't try again as it will get very
3195                 # expensive repeatedly fetching all the files for labels that will
3196                 # never be imported. If the label is moved in the future, the
3197                 # ignore will need to be removed manually.
3198                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3199
3200     def guessProjectName(self):
3201         for p in self.depotPaths:
3202             if p.endswith("/"):
3203                 p = p[:-1]
3204             p = p[p.strip().rfind("/") + 1:]
3205             if not p.endswith("/"):
3206                p += "/"
3207             return p
3208
3209     def getBranchMapping(self):
3210         lostAndFoundBranches = set()
3211
3212         user = gitConfig("git-p4.branchUser")
3213         if len(user) > 0:
3214             command = "branches -u %s" % user
3215         else:
3216             command = "branches"
3217
3218         for info in p4CmdList(command):
3219             details = p4Cmd(["branch", "-o", info["branch"]])
3220             viewIdx = 0
3221             while "View%s" % viewIdx in details:
3222                 paths = details["View%s" % viewIdx].split(" ")
3223                 viewIdx = viewIdx + 1
3224                 # require standard //depot/foo/... //depot/bar/... mapping
3225                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3226                     continue
3227                 source = paths[0]
3228                 destination = paths[1]
3229                 ## HACK
3230                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3231                     source = source[len(self.depotPaths[0]):-4]
3232                     destination = destination[len(self.depotPaths[0]):-4]
3233
3234                     if destination in self.knownBranches:
3235                         if not self.silent:
3236                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3237                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3238                         continue
3239
3240                     self.knownBranches[destination] = source
3241
3242                     lostAndFoundBranches.discard(destination)
3243
3244                     if source not in self.knownBranches:
3245                         lostAndFoundBranches.add(source)
3246
3247         # Perforce does not strictly require branches to be defined, so we also
3248         # check git config for a branch list.
3249         #
3250         # Example of branch definition in git config file:
3251         # [git-p4]
3252         #   branchList=main:branchA
3253         #   branchList=main:branchB
3254         #   branchList=branchA:branchC
3255         configBranches = gitConfigList("git-p4.branchList")
3256         for branch in configBranches:
3257             if branch:
3258                 (source, destination) = branch.split(":")
3259                 self.knownBranches[destination] = source
3260
3261                 lostAndFoundBranches.discard(destination)
3262
3263                 if source not in self.knownBranches:
3264                     lostAndFoundBranches.add(source)
3265
3266
3267         for branch in lostAndFoundBranches:
3268             self.knownBranches[branch] = branch
3269
3270     def getBranchMappingFromGitBranches(self):
3271         branches = p4BranchesInGit(self.importIntoRemotes)
3272         for branch in branches.keys():
3273             if branch == "master":
3274                 branch = "main"
3275             else:
3276                 branch = branch[len(self.projectName):]
3277             self.knownBranches[branch] = branch
3278
3279     def updateOptionDict(self, d):
3280         option_keys = {}
3281         if self.keepRepoPath:
3282             option_keys['keepRepoPath'] = 1
3283
3284         d["options"] = ' '.join(sorted(option_keys.keys()))
3285
3286     def readOptions(self, d):
3287         self.keepRepoPath = ('options' in d
3288                              and ('keepRepoPath' in d['options']))
3289
3290     def gitRefForBranch(self, branch):
3291         if branch == "main":
3292             return self.refPrefix + "master"
3293
3294         if len(branch) <= 0:
3295             return branch
3296
3297         return self.refPrefix + self.projectName + branch
3298
3299     def gitCommitByP4Change(self, ref, change):
3300         if self.verbose:
3301             print("looking in ref " + ref + " for change %s using bisect..." % change)
3302
3303         earliestCommit = ""
3304         latestCommit = parseRevision(ref)
3305
3306         while True:
3307             if self.verbose:
3308                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3309             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3310             if len(next) == 0:
3311                 if self.verbose:
3312                     print("argh")
3313                 return ""
3314             log = extractLogMessageFromGitCommit(next)
3315             settings = extractSettingsGitLog(log)
3316             currentChange = int(settings['change'])
3317             if self.verbose:
3318                 print("current change %s" % currentChange)
3319
3320             if currentChange == change:
3321                 if self.verbose:
3322                     print("found %s" % next)
3323                 return next
3324
3325             if currentChange < change:
3326                 earliestCommit = "^%s" % next
3327             else:
3328                 latestCommit = "%s" % next
3329
3330         return ""
3331
3332     def importNewBranch(self, branch, maxChange):
3333         # make fast-import flush all changes to disk and update the refs using the checkpoint
3334         # command so that we can try to find the branch parent in the git history
3335         self.gitStream.write("checkpoint\n\n");
3336         self.gitStream.flush();
3337         branchPrefix = self.depotPaths[0] + branch + "/"
3338         range = "@1,%s" % maxChange
3339         #print "prefix" + branchPrefix
3340         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3341         if len(changes) <= 0:
3342             return False
3343         firstChange = changes[0]
3344         #print "first change in branch: %s" % firstChange
3345         sourceBranch = self.knownBranches[branch]
3346         sourceDepotPath = self.depotPaths[0] + sourceBranch
3347         sourceRef = self.gitRefForBranch(sourceBranch)
3348         #print "source " + sourceBranch
3349
3350         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3351         #print "branch parent: %s" % branchParentChange
3352         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3353         if len(gitParent) > 0:
3354             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3355             #print "parent git commit: %s" % gitParent
3356
3357         self.importChanges(changes)
3358         return True
3359
3360     def searchParent(self, parent, branch, target):
3361         parentFound = False
3362         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3363                                      "--no-merges", parent]):
3364             blob = blob.strip()
3365             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3366                 parentFound = True
3367                 if self.verbose:
3368                     print("Found parent of %s in commit %s" % (branch, blob))
3369                 break
3370         if parentFound:
3371             return blob
3372         else:
3373             return None
3374
3375     def importChanges(self, changes, origin_revision=0):
3376         cnt = 1
3377         for change in changes:
3378             description = p4_describe(change)
3379             self.updateOptionDict(description)
3380
3381             if not self.silent:
3382                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3383                 sys.stdout.flush()
3384             cnt = cnt + 1
3385
3386             try:
3387                 if self.detectBranches:
3388                     branches = self.splitFilesIntoBranches(description)
3389                     for branch in branches.keys():
3390                         ## HACK  --hwn
3391                         branchPrefix = self.depotPaths[0] + branch + "/"
3392                         self.branchPrefixes = [ branchPrefix ]
3393
3394                         parent = ""
3395
3396                         filesForCommit = branches[branch]
3397
3398                         if self.verbose:
3399                             print("branch is %s" % branch)
3400
3401                         self.updatedBranches.add(branch)
3402
3403                         if branch not in self.createdBranches:
3404                             self.createdBranches.add(branch)
3405                             parent = self.knownBranches[branch]
3406                             if parent == branch:
3407                                 parent = ""
3408                             else:
3409                                 fullBranch = self.projectName + branch
3410                                 if fullBranch not in self.p4BranchesInGit:
3411                                     if not self.silent:
3412                                         print("\n    Importing new branch %s" % fullBranch);
3413                                     if self.importNewBranch(branch, change - 1):
3414                                         parent = ""
3415                                         self.p4BranchesInGit.append(fullBranch)
3416                                     if not self.silent:
3417                                         print("\n    Resuming with change %s" % change);
3418
3419                                 if self.verbose:
3420                                     print("parent determined through known branches: %s" % parent)
3421
3422                         branch = self.gitRefForBranch(branch)
3423                         parent = self.gitRefForBranch(parent)
3424
3425                         if self.verbose:
3426                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3427
3428                         if len(parent) == 0 and branch in self.initialParents:
3429                             parent = self.initialParents[branch]
3430                             del self.initialParents[branch]
3431
3432                         blob = None
3433                         if len(parent) > 0:
3434                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3435                             if self.verbose:
3436                                 print("Creating temporary branch: " + tempBranch)
3437                             self.commit(description, filesForCommit, tempBranch)
3438                             self.tempBranches.append(tempBranch)
3439                             self.checkpoint()
3440                             blob = self.searchParent(parent, branch, tempBranch)
3441                         if blob:
3442                             self.commit(description, filesForCommit, branch, blob)
3443                         else:
3444                             if self.verbose:
3445                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3446                             self.commit(description, filesForCommit, branch, parent)
3447                 else:
3448                     files = self.extractFilesFromCommit(description)
3449                     self.commit(description, files, self.branch,
3450                                 self.initialParent)
3451                     # only needed once, to connect to the previous commit
3452                     self.initialParent = ""
3453             except IOError:
3454                 print(self.gitError.read())
3455                 sys.exit(1)
3456
3457     def sync_origin_only(self):
3458         if self.syncWithOrigin:
3459             self.hasOrigin = originP4BranchesExist()
3460             if self.hasOrigin:
3461                 if not self.silent:
3462                     print('Syncing with origin first, using "git fetch origin"')
3463                 system("git fetch origin")
3464
3465     def importHeadRevision(self, revision):
3466         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3467
3468         details = {}
3469         details["user"] = "git perforce import user"
3470         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3471                            % (' '.join(self.depotPaths), revision))
3472         details["change"] = revision
3473         newestRevision = 0
3474
3475         fileCnt = 0
3476         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3477
3478         for info in p4CmdList(["files"] + fileArgs):
3479
3480             if 'code' in info and info['code'] == 'error':
3481                 sys.stderr.write("p4 returned an error: %s\n"
3482                                  % info['data'])
3483                 if info['data'].find("must refer to client") >= 0:
3484                     sys.stderr.write("This particular p4 error is misleading.\n")
3485                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3486                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3487                 sys.exit(1)
3488             if 'p4ExitCode' in info:
3489                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3490                 sys.exit(1)
3491
3492
3493             change = int(info["change"])
3494             if change > newestRevision:
3495                 newestRevision = change
3496
3497             if info["action"] in self.delete_actions:
3498                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3499                 #fileCnt = fileCnt + 1
3500                 continue
3501
3502             for prop in ["depotFile", "rev", "action", "type" ]:
3503                 details["%s%s" % (prop, fileCnt)] = info[prop]
3504
3505             fileCnt = fileCnt + 1
3506
3507         details["change"] = newestRevision
3508
3509         # Use time from top-most change so that all git p4 clones of
3510         # the same p4 repo have the same commit SHA1s.
3511         res = p4_describe(newestRevision)
3512         details["time"] = res["time"]
3513
3514         self.updateOptionDict(details)
3515         try:
3516             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3517         except IOError:
3518             print("IO error with git fast-import. Is your git version recent enough?")
3519             print(self.gitError.read())
3520
3521     def openStreams(self):
3522         self.importProcess = subprocess.Popen(["git", "fast-import"],
3523                                               stdin=subprocess.PIPE,
3524                                               stdout=subprocess.PIPE,
3525                                               stderr=subprocess.PIPE);
3526         self.gitOutput = self.importProcess.stdout
3527         self.gitStream = self.importProcess.stdin
3528         self.gitError = self.importProcess.stderr
3529
3530     def closeStreams(self):
3531         self.gitStream.close()
3532         if self.importProcess.wait() != 0:
3533             die("fast-import failed: %s" % self.gitError.read())
3534         self.gitOutput.close()
3535         self.gitError.close()
3536
3537     def run(self, args):
3538         if self.importIntoRemotes:
3539             self.refPrefix = "refs/remotes/p4/"
3540         else:
3541             self.refPrefix = "refs/heads/p4/"
3542
3543         self.sync_origin_only()
3544
3545         branch_arg_given = bool(self.branch)
3546         if len(self.branch) == 0:
3547             self.branch = self.refPrefix + "master"
3548             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3549                 system("git update-ref %s refs/heads/p4" % self.branch)
3550                 system("git branch -D p4")
3551
3552         # accept either the command-line option, or the configuration variable
3553         if self.useClientSpec:
3554             # will use this after clone to set the variable
3555             self.useClientSpec_from_options = True
3556         else:
3557             if gitConfigBool("git-p4.useclientspec"):
3558                 self.useClientSpec = True
3559         if self.useClientSpec:
3560             self.clientSpecDirs = getClientSpec()
3561
3562         # TODO: should always look at previous commits,
3563         # merge with previous imports, if possible.
3564         if args == []:
3565             if self.hasOrigin:
3566                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3567
3568             # branches holds mapping from branch name to sha1
3569             branches = p4BranchesInGit(self.importIntoRemotes)
3570
3571             # restrict to just this one, disabling detect-branches
3572             if branch_arg_given:
3573                 short = self.branch.split("/")[-1]
3574                 if short in branches:
3575                     self.p4BranchesInGit = [ short ]
3576             else:
3577                 self.p4BranchesInGit = branches.keys()
3578
3579             if len(self.p4BranchesInGit) > 1:
3580                 if not self.silent:
3581                     print("Importing from/into multiple branches")
3582                 self.detectBranches = True
3583                 for branch in branches.keys():
3584                     self.initialParents[self.refPrefix + branch] = \
3585                         branches[branch]
3586
3587             if self.verbose:
3588                 print("branches: %s" % self.p4BranchesInGit)
3589
3590             p4Change = 0
3591             for branch in self.p4BranchesInGit:
3592                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3593
3594                 settings = extractSettingsGitLog(logMsg)
3595
3596                 self.readOptions(settings)
3597                 if ('depot-paths' in settings
3598                     and 'change' in settings):
3599                     change = int(settings['change']) + 1
3600                     p4Change = max(p4Change, change)
3601
3602                     depotPaths = sorted(settings['depot-paths'])
3603                     if self.previousDepotPaths == []:
3604                         self.previousDepotPaths = depotPaths
3605                     else:
3606                         paths = []
3607                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3608                             prev_list = prev.split("/")
3609                             cur_list = cur.split("/")
3610                             for i in range(0, min(len(cur_list), len(prev_list))):
3611                                 if cur_list[i] != prev_list[i]:
3612                                     i = i - 1
3613                                     break
3614
3615                             paths.append ("/".join(cur_list[:i + 1]))
3616
3617                         self.previousDepotPaths = paths
3618
3619             if p4Change > 0:
3620                 self.depotPaths = sorted(self.previousDepotPaths)
3621                 self.changeRange = "@%s,#head" % p4Change
3622                 if not self.silent and not self.detectBranches:
3623                     print("Performing incremental import into %s git branch" % self.branch)
3624
3625         # accept multiple ref name abbreviations:
3626         #    refs/foo/bar/branch -> use it exactly
3627         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3628         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3629         if not self.branch.startswith("refs/"):
3630             if self.importIntoRemotes:
3631                 prepend = "refs/remotes/"
3632             else:
3633                 prepend = "refs/heads/"
3634             if not self.branch.startswith("p4/"):
3635                 prepend += "p4/"
3636             self.branch = prepend + self.branch
3637
3638         if len(args) == 0 and self.depotPaths:
3639             if not self.silent:
3640                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3641         else:
3642             if self.depotPaths and self.depotPaths != args:
3643                 print("previous import used depot path %s and now %s was specified. "
3644                        "This doesn't work!" % (' '.join (self.depotPaths),
3645                                                ' '.join (args)))
3646                 sys.exit(1)
3647
3648             self.depotPaths = sorted(args)
3649
3650         revision = ""
3651         self.users = {}
3652
3653         # Make sure no revision specifiers are used when --changesfile
3654         # is specified.
3655         bad_changesfile = False
3656         if len(self.changesFile) > 0:
3657             for p in self.depotPaths:
3658                 if p.find("@") >= 0 or p.find("#") >= 0:
3659                     bad_changesfile = True
3660                     break
3661         if bad_changesfile:
3662             die("Option --changesfile is incompatible with revision specifiers")
3663
3664         newPaths = []
3665         for p in self.depotPaths:
3666             if p.find("@") != -1:
3667                 atIdx = p.index("@")
3668                 self.changeRange = p[atIdx:]
3669                 if self.changeRange == "@all":
3670                     self.changeRange = ""
3671                 elif ',' not in self.changeRange:
3672                     revision = self.changeRange
3673                     self.changeRange = ""
3674                 p = p[:atIdx]
3675             elif p.find("#") != -1:
3676                 hashIdx = p.index("#")
3677                 revision = p[hashIdx:]
3678                 p = p[:hashIdx]
3679             elif self.previousDepotPaths == []:
3680                 # pay attention to changesfile, if given, else import
3681                 # the entire p4 tree at the head revision
3682                 if len(self.changesFile) == 0:
3683                     revision = "#head"
3684
3685             p = re.sub ("\.\.\.$", "", p)
3686             if not p.endswith("/"):
3687                 p += "/"
3688
3689             newPaths.append(p)
3690
3691         self.depotPaths = newPaths
3692
3693         # --detect-branches may change this for each branch
3694         self.branchPrefixes = self.depotPaths
3695
3696         self.loadUserMapFromCache()
3697         self.labels = {}
3698         if self.detectLabels:
3699             self.getLabels();
3700
3701         if self.detectBranches:
3702             ## FIXME - what's a P4 projectName ?
3703             self.projectName = self.guessProjectName()
3704
3705             if self.hasOrigin:
3706                 self.getBranchMappingFromGitBranches()
3707             else:
3708                 self.getBranchMapping()
3709             if self.verbose:
3710                 print("p4-git branches: %s" % self.p4BranchesInGit)
3711                 print("initial parents: %s" % self.initialParents)
3712             for b in self.p4BranchesInGit:
3713                 if b != "master":
3714
3715                     ## FIXME
3716                     b = b[len(self.projectName):]
3717                 self.createdBranches.add(b)
3718
3719         self.openStreams()
3720
3721         if revision:
3722             self.importHeadRevision(revision)
3723         else:
3724             changes = []
3725
3726             if len(self.changesFile) > 0:
3727                 output = open(self.changesFile).readlines()
3728                 changeSet = set()
3729                 for line in output:
3730                     changeSet.add(int(line))
3731
3732                 for change in changeSet:
3733                     changes.append(change)
3734
3735                 changes.sort()
3736             else:
3737                 # catch "git p4 sync" with no new branches, in a repo that
3738                 # does not have any existing p4 branches
3739                 if len(args) == 0:
3740                     if not self.p4BranchesInGit:
3741                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3742
3743                     # The default branch is master, unless --branch is used to
3744                     # specify something else.  Make sure it exists, or complain
3745                     # nicely about how to use --branch.
3746                     if not self.detectBranches:
3747                         if not branch_exists(self.branch):
3748                             if branch_arg_given:
3749                                 die("Error: branch %s does not exist." % self.branch)
3750                             else:
3751                                 die("Error: no branch %s; perhaps specify one with --branch." %
3752                                     self.branch)
3753
3754                 if self.verbose:
3755                     print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3756                                                               self.changeRange))
3757                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3758
3759                 if len(self.maxChanges) > 0:
3760                     changes = changes[:min(int(self.maxChanges), len(changes))]
3761
3762             if len(changes) == 0:
3763                 if not self.silent:
3764                     print("No changes to import!")
3765             else:
3766                 if not self.silent and not self.detectBranches:
3767                     print("Import destination: %s" % self.branch)
3768
3769                 self.updatedBranches = set()
3770
3771                 if not self.detectBranches:
3772                     if args:
3773                         # start a new branch
3774                         self.initialParent = ""
3775                     else:
3776                         # build on a previous revision
3777                         self.initialParent = parseRevision(self.branch)
3778
3779                 self.importChanges(changes)
3780
3781                 if not self.silent:
3782                     print("")
3783                     if len(self.updatedBranches) > 0:
3784                         sys.stdout.write("Updated branches: ")
3785                         for b in self.updatedBranches:
3786                             sys.stdout.write("%s " % b)
3787                         sys.stdout.write("\n")
3788
3789         if gitConfigBool("git-p4.importLabels"):
3790             self.importLabels = True
3791
3792         if self.importLabels:
3793             p4Labels = getP4Labels(self.depotPaths)
3794             gitTags = getGitTags()
3795
3796             missingP4Labels = p4Labels - gitTags
3797             self.importP4Labels(self.gitStream, missingP4Labels)
3798
3799         self.closeStreams()
3800
3801         # Cleanup temporary branches created during import
3802         if self.tempBranches != []:
3803             for branch in self.tempBranches:
3804                 read_pipe("git update-ref -d %s" % branch)
3805             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3806
3807         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3808         # a convenient shortcut refname "p4".
3809         if self.importIntoRemotes:
3810             head_ref = self.refPrefix + "HEAD"
3811             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3812                 system(["git", "symbolic-ref", head_ref, self.branch])
3813
3814         return True
3815
3816 class P4Rebase(Command):
3817     def __init__(self):
3818         Command.__init__(self)
3819         self.options = [
3820                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3821         ]
3822         self.importLabels = False
3823         self.description = ("Fetches the latest revision from perforce and "
3824                             + "rebases the current work (branch) against it")
3825
3826     def run(self, args):
3827         sync = P4Sync()
3828         sync.importLabels = self.importLabels
3829         sync.run([])
3830
3831         return self.rebase()
3832
3833     def rebase(self):
3834         if os.system("git update-index --refresh") != 0:
3835             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3836         if len(read_pipe("git diff-index HEAD --")) > 0:
3837             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3838
3839         [upstream, settings] = findUpstreamBranchPoint()
3840         if len(upstream) == 0:
3841             die("Cannot find upstream branchpoint for rebase")
3842
3843         # the branchpoint may be p4/foo~3, so strip off the parent
3844         upstream = re.sub("~[0-9]+$", "", upstream)
3845
3846         print("Rebasing the current branch onto %s" % upstream)
3847         oldHead = read_pipe("git rev-parse HEAD").strip()
3848         system("git rebase %s" % upstream)
3849         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3850         return True
3851
3852 class P4Clone(P4Sync):
3853     def __init__(self):
3854         P4Sync.__init__(self)
3855         self.description = "Creates a new git repository and imports from Perforce into it"
3856         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3857         self.options += [
3858             optparse.make_option("--destination", dest="cloneDestination",
3859                                  action='store', default=None,
3860                                  help="where to leave result of the clone"),
3861             optparse.make_option("--bare", dest="cloneBare",
3862                                  action="store_true", default=False),
3863         ]
3864         self.cloneDestination = None
3865         self.needsGit = False
3866         self.cloneBare = False
3867
3868     def defaultDestination(self, args):
3869         ## TODO: use common prefix of args?
3870         depotPath = args[0]
3871         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3872         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3873         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3874         depotDir = re.sub(r"/$", "", depotDir)
3875         return os.path.split(depotDir)[1]
3876
3877     def run(self, args):
3878         if len(args) < 1:
3879             return False
3880
3881         if self.keepRepoPath and not self.cloneDestination:
3882             sys.stderr.write("Must specify destination for --keep-path\n")
3883             sys.exit(1)
3884
3885         depotPaths = args
3886
3887         if not self.cloneDestination and len(depotPaths) > 1:
3888             self.cloneDestination = depotPaths[-1]
3889             depotPaths = depotPaths[:-1]
3890
3891         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3892         for p in depotPaths:
3893             if not p.startswith("//"):
3894                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3895                 return False
3896
3897         if not self.cloneDestination:
3898             self.cloneDestination = self.defaultDestination(args)
3899
3900         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3901
3902         if not os.path.exists(self.cloneDestination):
3903             os.makedirs(self.cloneDestination)
3904         chdir(self.cloneDestination)
3905
3906         init_cmd = [ "git", "init" ]
3907         if self.cloneBare:
3908             init_cmd.append("--bare")
3909         retcode = subprocess.call(init_cmd)
3910         if retcode:
3911             raise CalledProcessError(retcode, init_cmd)
3912
3913         if not P4Sync.run(self, depotPaths):
3914             return False
3915
3916         # create a master branch and check out a work tree
3917         if gitBranchExists(self.branch):
3918             system([ "git", "branch", "master", self.branch ])
3919             if not self.cloneBare:
3920                 system([ "git", "checkout", "-f" ])
3921         else:
3922             print('Not checking out any branch, use ' \
3923                   '"git checkout -q -b master <branch>"')
3924
3925         # auto-set this variable if invoked with --use-client-spec
3926         if self.useClientSpec_from_options:
3927             system("git config --bool git-p4.useclientspec true")
3928
3929         return True
3930
3931 class P4Unshelve(Command):
3932     def __init__(self):
3933         Command.__init__(self)
3934         self.options = []
3935         self.origin = "HEAD"
3936         self.description = "Unshelve a P4 changelist into a git commit"
3937         self.usage = "usage: %prog [options] changelist"
3938         self.options += [
3939                 optparse.make_option("--origin", dest="origin",
3940                     help="Use this base revision instead of the default (%s)" % self.origin),
3941         ]
3942         self.verbose = False
3943         self.noCommit = False
3944         self.destbranch = "refs/remotes/p4-unshelved"
3945
3946     def renameBranch(self, branch_name):
3947         """ Rename the existing branch to branch_name.N
3948         """
3949
3950         found = True
3951         for i in range(0,1000):
3952             backup_branch_name = "{0}.{1}".format(branch_name, i)
3953             if not gitBranchExists(backup_branch_name):
3954                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3955                 gitDeleteRef(branch_name)
3956                 found = True
3957                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3958                 break
3959
3960         if not found:
3961             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3962
3963     def findLastP4Revision(self, starting_point):
3964         """ Look back from starting_point for the first commit created by git-p4
3965             to find the P4 commit we are based on, and the depot-paths.
3966         """
3967
3968         for parent in (range(65535)):
3969             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3970             settings = extractSettingsGitLog(log)
3971             if 'change' in settings:
3972                 return settings
3973
3974         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3975
3976     def createShelveParent(self, change, branch_name, sync, origin):
3977         """ Create a commit matching the parent of the shelved changelist 'change'
3978         """
3979         parent_description = p4_describe(change, shelved=True)
3980         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
3981         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
3982
3983         parent_files = []
3984         for f in files:
3985             # if it was added in the shelved changelist, it won't exist in the parent
3986             if f['action'] in self.add_actions:
3987                 continue
3988
3989             # if it was deleted in the shelved changelist it must not be deleted
3990             # in the parent - we might even need to create it if the origin branch
3991             # does not have it
3992             if f['action'] in self.delete_actions:
3993                 f['action'] = 'add'
3994
3995             parent_files.append(f)
3996
3997         sync.commit(parent_description, parent_files, branch_name,
3998                 parent=origin, allow_empty=True)
3999         print("created parent commit for {0} based on {1} in {2}".format(
4000             change, self.origin, branch_name))
4001
4002     def run(self, args):
4003         if len(args) != 1:
4004             return False
4005
4006         if not gitBranchExists(self.origin):
4007             sys.exit("origin branch {0} does not exist".format(self.origin))
4008
4009         sync = P4Sync()
4010         changes = args
4011
4012         # only one change at a time
4013         change = changes[0]
4014
4015         # if the target branch already exists, rename it
4016         branch_name = "{0}/{1}".format(self.destbranch, change)
4017         if gitBranchExists(branch_name):
4018             self.renameBranch(branch_name)
4019         sync.branch = branch_name
4020
4021         sync.verbose = self.verbose
4022         sync.suppress_meta_comment = True
4023
4024         settings = self.findLastP4Revision(self.origin)
4025         sync.depotPaths = settings['depot-paths']
4026         sync.branchPrefixes = sync.depotPaths
4027
4028         sync.openStreams()
4029         sync.loadUserMapFromCache()
4030         sync.silent = True
4031
4032         # create a commit for the parent of the shelved changelist
4033         self.createShelveParent(change, branch_name, sync, self.origin)
4034
4035         # create the commit for the shelved changelist itself
4036         description = p4_describe(change, True)
4037         files = sync.extractFilesFromCommit(description, True, change)
4038
4039         sync.commit(description, files, branch_name, "")
4040         sync.closeStreams()
4041
4042         print("unshelved changelist {0} into {1}".format(change, branch_name))
4043
4044         return True
4045
4046 class P4Branches(Command):
4047     def __init__(self):
4048         Command.__init__(self)
4049         self.options = [ ]
4050         self.description = ("Shows the git branches that hold imports and their "
4051                             + "corresponding perforce depot paths")
4052         self.verbose = False
4053
4054     def run(self, args):
4055         if originP4BranchesExist():
4056             createOrUpdateBranchesFromOrigin()
4057
4058         cmdline = "git rev-parse --symbolic "
4059         cmdline += " --remotes"
4060
4061         for line in read_pipe_lines(cmdline):
4062             line = line.strip()
4063
4064             if not line.startswith('p4/') or line == "p4/HEAD":
4065                 continue
4066             branch = line
4067
4068             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4069             settings = extractSettingsGitLog(log)
4070
4071             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4072         return True
4073
4074 class HelpFormatter(optparse.IndentedHelpFormatter):
4075     def __init__(self):
4076         optparse.IndentedHelpFormatter.__init__(self)
4077
4078     def format_description(self, description):
4079         if description:
4080             return description + "\n"
4081         else:
4082             return ""
4083
4084 def printUsage(commands):
4085     print("usage: %s <command> [options]" % sys.argv[0])
4086     print("")
4087     print("valid commands: %s" % ", ".join(commands))
4088     print("")
4089     print("Try %s <command> --help for command specific help." % sys.argv[0])
4090     print("")
4091
4092 commands = {
4093     "debug" : P4Debug,
4094     "submit" : P4Submit,
4095     "commit" : P4Submit,
4096     "sync" : P4Sync,
4097     "rebase" : P4Rebase,
4098     "clone" : P4Clone,
4099     "rollback" : P4RollBack,
4100     "branches" : P4Branches,
4101     "unshelve" : P4Unshelve,
4102 }
4103
4104
4105 def main():
4106     if len(sys.argv[1:]) == 0:
4107         printUsage(commands.keys())
4108         sys.exit(2)
4109
4110     cmdName = sys.argv[1]
4111     try:
4112         klass = commands[cmdName]
4113         cmd = klass()
4114     except KeyError:
4115         print("unknown command %s" % cmdName)
4116         print("")
4117         printUsage(commands.keys())
4118         sys.exit(2)
4119
4120     options = cmd.options
4121     cmd.gitdir = os.environ.get("GIT_DIR", None)
4122
4123     args = sys.argv[2:]
4124
4125     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4126     if cmd.needsGit:
4127         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4128
4129     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4130                                    options,
4131                                    description = cmd.description,
4132                                    formatter = HelpFormatter())
4133
4134     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4135     global verbose
4136     verbose = cmd.verbose
4137     if cmd.needsGit:
4138         if cmd.gitdir == None:
4139             cmd.gitdir = os.path.abspath(".git")
4140             if not isValidGitDir(cmd.gitdir):
4141                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4142                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4143                 if os.path.exists(cmd.gitdir):
4144                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4145                     if len(cdup) > 0:
4146                         chdir(cdup);
4147
4148         if not isValidGitDir(cmd.gitdir):
4149             if isValidGitDir(cmd.gitdir + "/.git"):
4150                 cmd.gitdir += "/.git"
4151             else:
4152                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4153
4154         # so git commands invoked from the P4 workspace will succeed
4155         os.environ["GIT_DIR"] = cmd.gitdir
4156
4157     if not cmd.run(args):
4158         parser.print_help()
4159         sys.exit(2)
4160
4161
4162 if __name__ == '__main__':
4163     main()