Merge branch 'sg/test-squelch-noise-in-commit-bulk'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 # support basestring in python3
31 try:
32     unicode = unicode
33 except NameError:
34     # 'unicode' is undefined, must be Python 3
35     str = str
36     unicode = str
37     bytes = bytes
38     basestring = (str,bytes)
39 else:
40     # 'unicode' exists, must be Python 2
41     str = str
42     unicode = unicode
43     bytes = str
44     basestring = basestring
45
46 try:
47     from subprocess import CalledProcessError
48 except ImportError:
49     # from python2.7:subprocess.py
50     # Exception classes used by this module.
51     class CalledProcessError(Exception):
52         """This exception is raised when a process run by check_call() returns
53         a non-zero exit status.  The exit status will be stored in the
54         returncode attribute."""
55         def __init__(self, returncode, cmd):
56             self.returncode = returncode
57             self.cmd = cmd
58         def __str__(self):
59             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
61 verbose = False
62
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
65
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
68
69 p4_access_checked = False
70
71 def p4_build_cmd(cmd):
72     """Build a suitable p4 command line.
73
74     This consolidates building and returning a p4 command line into one
75     location. It means that hooking into the environment, or other configuration
76     can be done more easily.
77     """
78     real_cmd = ["p4"]
79
80     user = gitConfig("git-p4.user")
81     if len(user) > 0:
82         real_cmd += ["-u",user]
83
84     password = gitConfig("git-p4.password")
85     if len(password) > 0:
86         real_cmd += ["-P", password]
87
88     port = gitConfig("git-p4.port")
89     if len(port) > 0:
90         real_cmd += ["-p", port]
91
92     host = gitConfig("git-p4.host")
93     if len(host) > 0:
94         real_cmd += ["-H", host]
95
96     client = gitConfig("git-p4.client")
97     if len(client) > 0:
98         real_cmd += ["-c", client]
99
100     retries = gitConfigInt("git-p4.retries")
101     if retries is None:
102         # Perform 3 retries by default
103         retries = 3
104     if retries > 0:
105         # Provide a way to not pass this option by setting git-p4.retries to 0
106         real_cmd += ["-r", str(retries)]
107
108     if isinstance(cmd,basestring):
109         real_cmd = ' '.join(real_cmd) + ' ' + cmd
110     else:
111         real_cmd += cmd
112
113     # now check that we can actually talk to the server
114     global p4_access_checked
115     if not p4_access_checked:
116         p4_access_checked = True    # suppress access checks in p4_check_access itself
117         p4_check_access()
118
119     return real_cmd
120
121 def git_dir(path):
122     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123         This won't automatically add ".git" to a directory.
124     """
125     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126     if not d or len(d) == 0:
127         return None
128     else:
129         return d
130
131 def chdir(path, is_client_path=False):
132     """Do chdir to the given path, and set the PWD environment
133        variable for use by P4.  It does not look at getcwd() output.
134        Since we're not using the shell, it is necessary to set the
135        PWD environment variable explicitly.
136
137        Normally, expand the path to force it to be absolute.  This
138        addresses the use of relative path names inside P4 settings,
139        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
140        as given; it looks for .p4config using PWD.
141
142        If is_client_path, the path was handed to us directly by p4,
143        and may be a symbolic link.  Do not call os.getcwd() in this
144        case, because it will cause p4 to think that PWD is not inside
145        the client path.
146        """
147
148     os.chdir(path)
149     if not is_client_path:
150         path = os.getcwd()
151     os.environ['PWD'] = path
152
153 def calcDiskFree():
154     """Return free space in bytes on the disk of the given dirname."""
155     if platform.system() == 'Windows':
156         free_bytes = ctypes.c_ulonglong(0)
157         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158         return free_bytes.value
159     else:
160         st = os.statvfs(os.getcwd())
161         return st.f_bavail * st.f_frsize
162
163 def die(msg):
164     if verbose:
165         raise Exception(msg)
166     else:
167         sys.stderr.write(msg + "\n")
168         sys.exit(1)
169
170 def write_pipe(c, stdin):
171     if verbose:
172         sys.stderr.write('Writing pipe: %s\n' % str(c))
173
174     expand = isinstance(c,basestring)
175     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176     pipe = p.stdin
177     val = pipe.write(stdin)
178     pipe.close()
179     if p.wait():
180         die('Command failed: %s' % str(c))
181
182     return val
183
184 def p4_write_pipe(c, stdin):
185     real_cmd = p4_build_cmd(c)
186     return write_pipe(real_cmd, stdin)
187
188 def read_pipe_full(c):
189     """ Read output from  command. Returns a tuple
190         of the return status, stdout text and stderr
191         text.
192     """
193     if verbose:
194         sys.stderr.write('Reading pipe: %s\n' % str(c))
195
196     expand = isinstance(c,basestring)
197     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198     (out, err) = p.communicate()
199     return (p.returncode, out, err)
200
201 def read_pipe(c, ignore_error=False):
202     """ Read output from  command. Returns the output text on
203         success. On failure, terminates execution, unless
204         ignore_error is True, when it returns an empty string.
205     """
206     (retcode, out, err) = read_pipe_full(c)
207     if retcode != 0:
208         if ignore_error:
209             out = ""
210         else:
211             die('Command failed: %s\nError: %s' % (str(c), err))
212     return out
213
214 def read_pipe_text(c):
215     """ Read output from a command with trailing whitespace stripped.
216         On error, returns None.
217     """
218     (retcode, out, err) = read_pipe_full(c)
219     if retcode != 0:
220         return None
221     else:
222         return out.rstrip()
223
224 def p4_read_pipe(c, ignore_error=False):
225     real_cmd = p4_build_cmd(c)
226     return read_pipe(real_cmd, ignore_error)
227
228 def read_pipe_lines(c):
229     if verbose:
230         sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232     expand = isinstance(c, basestring)
233     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234     pipe = p.stdout
235     val = pipe.readlines()
236     if pipe.close() or p.wait():
237         die('Command failed: %s' % str(c))
238
239     return val
240
241 def p4_read_pipe_lines(c):
242     """Specifically invoke p4 on the command supplied. """
243     real_cmd = p4_build_cmd(c)
244     return read_pipe_lines(real_cmd)
245
246 def p4_has_command(cmd):
247     """Ask p4 for help on this command.  If it returns an error, the
248        command does not exist in this version of p4."""
249     real_cmd = p4_build_cmd(["help", cmd])
250     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251                                    stderr=subprocess.PIPE)
252     p.communicate()
253     return p.returncode == 0
254
255 def p4_has_move_command():
256     """See if the move command exists, that it supports -k, and that
257        it has not been administratively disabled.  The arguments
258        must be correct, but the filenames do not have to exist.  Use
259        ones with wildcards so even if they exist, it will fail."""
260
261     if not p4_has_command("move"):
262         return False
263     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265     (out, err) = p.communicate()
266     # return code will be 1 in either case
267     if err.find("Invalid option") >= 0:
268         return False
269     if err.find("disabled") >= 0:
270         return False
271     # assume it failed because @... was invalid changelist
272     return True
273
274 def system(cmd, ignore_error=False):
275     expand = isinstance(cmd,basestring)
276     if verbose:
277         sys.stderr.write("executing %s\n" % str(cmd))
278     retcode = subprocess.call(cmd, shell=expand)
279     if retcode and not ignore_error:
280         raise CalledProcessError(retcode, cmd)
281
282     return retcode
283
284 def p4_system(cmd):
285     """Specifically invoke p4 as the system command. """
286     real_cmd = p4_build_cmd(cmd)
287     expand = isinstance(real_cmd, basestring)
288     retcode = subprocess.call(real_cmd, shell=expand)
289     if retcode:
290         raise CalledProcessError(retcode, real_cmd)
291
292 def die_bad_access(s):
293     die("failure accessing depot: {0}".format(s.rstrip()))
294
295 def p4_check_access(min_expiration=1):
296     """ Check if we can access Perforce - account still logged in
297     """
298     results = p4CmdList(["login", "-s"])
299
300     if len(results) == 0:
301         # should never get here: always get either some results, or a p4ExitCode
302         assert("could not parse response from perforce")
303
304     result = results[0]
305
306     if 'p4ExitCode' in result:
307         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308         die_bad_access("could not run p4")
309
310     code = result.get("code")
311     if not code:
312         # we get here if we couldn't connect and there was nothing to unmarshal
313         die_bad_access("could not connect")
314
315     elif code == "stat":
316         expiry = result.get("TicketExpiration")
317         if expiry:
318             expiry = int(expiry)
319             if expiry > min_expiration:
320                 # ok to carry on
321                 return
322             else:
323                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325         else:
326             # account without a timeout - all ok
327             return
328
329     elif code == "error":
330         data = result.get("data")
331         if data:
332             die_bad_access("p4 error: {0}".format(data))
333         else:
334             die_bad_access("unknown error")
335     elif code == "info":
336         return
337     else:
338         die_bad_access("unknown error code {0}".format(code))
339
340 _p4_version_string = None
341 def p4_version_string():
342     """Read the version string, showing just the last line, which
343        hopefully is the interesting version bit.
344
345        $ p4 -V
346        Perforce - The Fast Software Configuration Management System.
347        Copyright 1995-2011 Perforce Software.  All rights reserved.
348        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
349     """
350     global _p4_version_string
351     if not _p4_version_string:
352         a = p4_read_pipe_lines(["-V"])
353         _p4_version_string = a[-1].rstrip()
354     return _p4_version_string
355
356 def p4_integrate(src, dest):
357     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
358
359 def p4_sync(f, *options):
360     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
361
362 def p4_add(f):
363     # forcibly add file names with wildcards
364     if wildcard_present(f):
365         p4_system(["add", "-f", f])
366     else:
367         p4_system(["add", f])
368
369 def p4_delete(f):
370     p4_system(["delete", wildcard_encode(f)])
371
372 def p4_edit(f, *options):
373     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
374
375 def p4_revert(f):
376     p4_system(["revert", wildcard_encode(f)])
377
378 def p4_reopen(type, f):
379     p4_system(["reopen", "-t", type, wildcard_encode(f)])
380
381 def p4_reopen_in_change(changelist, files):
382     cmd = ["reopen", "-c", str(changelist)] + files
383     p4_system(cmd)
384
385 def p4_move(src, dest):
386     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
387
388 def p4_last_change():
389     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
390     return int(results[0]['change'])
391
392 def p4_describe(change, shelved=False):
393     """Make sure it returns a valid result by checking for
394        the presence of field "time".  Return a dict of the
395        results."""
396
397     cmd = ["describe", "-s"]
398     if shelved:
399         cmd += ["-S"]
400     cmd += [str(change)]
401
402     ds = p4CmdList(cmd, skip_info=True)
403     if len(ds) != 1:
404         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
405
406     d = ds[0]
407
408     if "p4ExitCode" in d:
409         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
410                                                       str(d)))
411     if "code" in d:
412         if d["code"] == "error":
413             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
414
415     if "time" not in d:
416         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
417
418     return d
419
420 #
421 # Canonicalize the p4 type and return a tuple of the
422 # base type, plus any modifiers.  See "p4 help filetypes"
423 # for a list and explanation.
424 #
425 def split_p4_type(p4type):
426
427     p4_filetypes_historical = {
428         "ctempobj": "binary+Sw",
429         "ctext": "text+C",
430         "cxtext": "text+Cx",
431         "ktext": "text+k",
432         "kxtext": "text+kx",
433         "ltext": "text+F",
434         "tempobj": "binary+FSw",
435         "ubinary": "binary+F",
436         "uresource": "resource+F",
437         "uxbinary": "binary+Fx",
438         "xbinary": "binary+x",
439         "xltext": "text+Fx",
440         "xtempobj": "binary+Swx",
441         "xtext": "text+x",
442         "xunicode": "unicode+x",
443         "xutf16": "utf16+x",
444     }
445     if p4type in p4_filetypes_historical:
446         p4type = p4_filetypes_historical[p4type]
447     mods = ""
448     s = p4type.split("+")
449     base = s[0]
450     mods = ""
451     if len(s) > 1:
452         mods = s[1]
453     return (base, mods)
454
455 #
456 # return the raw p4 type of a file (text, text+ko, etc)
457 #
458 def p4_type(f):
459     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
460     return results[0]['headType']
461
462 #
463 # Given a type base and modifier, return a regexp matching
464 # the keywords that can be expanded in the file
465 #
466 def p4_keywords_regexp_for_type(base, type_mods):
467     if base in ("text", "unicode", "binary"):
468         kwords = None
469         if "ko" in type_mods:
470             kwords = 'Id|Header'
471         elif "k" in type_mods:
472             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
473         else:
474             return None
475         pattern = r"""
476             \$              # Starts with a dollar, followed by...
477             (%s)            # one of the keywords, followed by...
478             (:[^$\n]+)?     # possibly an old expansion, followed by...
479             \$              # another dollar
480             """ % kwords
481         return pattern
482     else:
483         return None
484
485 #
486 # Given a file, return a regexp matching the possible
487 # RCS keywords that will be expanded, or None for files
488 # with kw expansion turned off.
489 #
490 def p4_keywords_regexp_for_file(file):
491     if not os.path.exists(file):
492         return None
493     else:
494         (type_base, type_mods) = split_p4_type(p4_type(file))
495         return p4_keywords_regexp_for_type(type_base, type_mods)
496
497 def setP4ExecBit(file, mode):
498     # Reopens an already open file and changes the execute bit to match
499     # the execute bit setting in the passed in mode.
500
501     p4Type = "+x"
502
503     if not isModeExec(mode):
504         p4Type = getP4OpenedType(file)
505         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
506         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
507         if p4Type[-1] == "+":
508             p4Type = p4Type[0:-1]
509
510     p4_reopen(p4Type, file)
511
512 def getP4OpenedType(file):
513     # Returns the perforce file type for the given file.
514
515     result = p4_read_pipe(["opened", wildcard_encode(file)])
516     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
517     if match:
518         return match.group(1)
519     else:
520         die("Could not determine file type for %s (result: '%s')" % (file, result))
521
522 # Return the set of all p4 labels
523 def getP4Labels(depotPaths):
524     labels = set()
525     if isinstance(depotPaths,basestring):
526         depotPaths = [depotPaths]
527
528     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
529         label = l['label']
530         labels.add(label)
531
532     return labels
533
534 # Return the set of all git tags
535 def getGitTags():
536     gitTags = set()
537     for line in read_pipe_lines(["git", "tag"]):
538         tag = line.strip()
539         gitTags.add(tag)
540     return gitTags
541
542 def diffTreePattern():
543     # This is a simple generator for the diff tree regex pattern. This could be
544     # a class variable if this and parseDiffTreeEntry were a part of a class.
545     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
546     while True:
547         yield pattern
548
549 def parseDiffTreeEntry(entry):
550     """Parses a single diff tree entry into its component elements.
551
552     See git-diff-tree(1) manpage for details about the format of the diff
553     output. This method returns a dictionary with the following elements:
554
555     src_mode - The mode of the source file
556     dst_mode - The mode of the destination file
557     src_sha1 - The sha1 for the source file
558     dst_sha1 - The sha1 fr the destination file
559     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
560     status_score - The score for the status (applicable for 'C' and 'R'
561                    statuses). This is None if there is no score.
562     src - The path for the source file.
563     dst - The path for the destination file. This is only present for
564           copy or renames. If it is not present, this is None.
565
566     If the pattern is not matched, None is returned."""
567
568     match = diffTreePattern().next().match(entry)
569     if match:
570         return {
571             'src_mode': match.group(1),
572             'dst_mode': match.group(2),
573             'src_sha1': match.group(3),
574             'dst_sha1': match.group(4),
575             'status': match.group(5),
576             'status_score': match.group(6),
577             'src': match.group(7),
578             'dst': match.group(10)
579         }
580     return None
581
582 def isModeExec(mode):
583     # Returns True if the given git mode represents an executable file,
584     # otherwise False.
585     return mode[-3:] == "755"
586
587 class P4Exception(Exception):
588     """ Base class for exceptions from the p4 client """
589     def __init__(self, exit_code):
590         self.p4ExitCode = exit_code
591
592 class P4ServerException(P4Exception):
593     """ Base class for exceptions where we get some kind of marshalled up result from the server """
594     def __init__(self, exit_code, p4_result):
595         super(P4ServerException, self).__init__(exit_code)
596         self.p4_result = p4_result
597         self.code = p4_result[0]['code']
598         self.data = p4_result[0]['data']
599
600 class P4RequestSizeException(P4ServerException):
601     """ One of the maxresults or maxscanrows errors """
602     def __init__(self, exit_code, p4_result, limit):
603         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
604         self.limit = limit
605
606 def isModeExecChanged(src_mode, dst_mode):
607     return isModeExec(src_mode) != isModeExec(dst_mode)
608
609 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
610         errors_as_exceptions=False):
611
612     if isinstance(cmd,basestring):
613         cmd = "-G " + cmd
614         expand = True
615     else:
616         cmd = ["-G"] + cmd
617         expand = False
618
619     cmd = p4_build_cmd(cmd)
620     if verbose:
621         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
622
623     # Use a temporary file to avoid deadlocks without
624     # subprocess.communicate(), which would put another copy
625     # of stdout into memory.
626     stdin_file = None
627     if stdin is not None:
628         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
629         if isinstance(stdin,basestring):
630             stdin_file.write(stdin)
631         else:
632             for i in stdin:
633                 stdin_file.write(i + '\n')
634         stdin_file.flush()
635         stdin_file.seek(0)
636
637     p4 = subprocess.Popen(cmd,
638                           shell=expand,
639                           stdin=stdin_file,
640                           stdout=subprocess.PIPE)
641
642     result = []
643     try:
644         while True:
645             entry = marshal.load(p4.stdout)
646             if skip_info:
647                 if 'code' in entry and entry['code'] == 'info':
648                     continue
649             if cb is not None:
650                 cb(entry)
651             else:
652                 result.append(entry)
653     except EOFError:
654         pass
655     exitCode = p4.wait()
656     if exitCode != 0:
657         if errors_as_exceptions:
658             if len(result) > 0:
659                 data = result[0].get('data')
660                 if data:
661                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
662                     if not m:
663                         m = re.search('Request too large \(over (\d+)\)', data)
664
665                     if m:
666                         limit = int(m.group(1))
667                         raise P4RequestSizeException(exitCode, result, limit)
668
669                 raise P4ServerException(exitCode, result)
670             else:
671                 raise P4Exception(exitCode)
672         else:
673             entry = {}
674             entry["p4ExitCode"] = exitCode
675             result.append(entry)
676
677     return result
678
679 def p4Cmd(cmd):
680     list = p4CmdList(cmd)
681     result = {}
682     for entry in list:
683         result.update(entry)
684     return result;
685
686 def p4Where(depotPath):
687     if not depotPath.endswith("/"):
688         depotPath += "/"
689     depotPathLong = depotPath + "..."
690     outputList = p4CmdList(["where", depotPathLong])
691     output = None
692     for entry in outputList:
693         if "depotFile" in entry:
694             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
695             # The base path always ends with "/...".
696             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
697                 output = entry
698                 break
699         elif "data" in entry:
700             data = entry.get("data")
701             space = data.find(" ")
702             if data[:space] == depotPath:
703                 output = entry
704                 break
705     if output == None:
706         return ""
707     if output["code"] == "error":
708         return ""
709     clientPath = ""
710     if "path" in output:
711         clientPath = output.get("path")
712     elif "data" in output:
713         data = output.get("data")
714         lastSpace = data.rfind(" ")
715         clientPath = data[lastSpace + 1:]
716
717     if clientPath.endswith("..."):
718         clientPath = clientPath[:-3]
719     return clientPath
720
721 def currentGitBranch():
722     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
723
724 def isValidGitDir(path):
725     return git_dir(path) != None
726
727 def parseRevision(ref):
728     return read_pipe("git rev-parse %s" % ref).strip()
729
730 def branchExists(ref):
731     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
732                      ignore_error=True)
733     return len(rev) > 0
734
735 def extractLogMessageFromGitCommit(commit):
736     logMessage = ""
737
738     ## fixme: title is first line of commit, not 1st paragraph.
739     foundTitle = False
740     for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
741        if not foundTitle:
742            if len(log) == 1:
743                foundTitle = True
744            continue
745
746        logMessage += log
747     return logMessage
748
749 def extractSettingsGitLog(log):
750     values = {}
751     for line in log.split("\n"):
752         line = line.strip()
753         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
754         if not m:
755             continue
756
757         assignments = m.group(1).split (':')
758         for a in assignments:
759             vals = a.split ('=')
760             key = vals[0].strip()
761             val = ('='.join (vals[1:])).strip()
762             if val.endswith ('\"') and val.startswith('"'):
763                 val = val[1:-1]
764
765             values[key] = val
766
767     paths = values.get("depot-paths")
768     if not paths:
769         paths = values.get("depot-path")
770     if paths:
771         values['depot-paths'] = paths.split(',')
772     return values
773
774 def gitBranchExists(branch):
775     proc = subprocess.Popen(["git", "rev-parse", branch],
776                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
777     return proc.wait() == 0;
778
779 def gitUpdateRef(ref, newvalue):
780     subprocess.check_call(["git", "update-ref", ref, newvalue])
781
782 def gitDeleteRef(ref):
783     subprocess.check_call(["git", "update-ref", "-d", ref])
784
785 _gitConfig = {}
786
787 def gitConfig(key, typeSpecifier=None):
788     if key not in _gitConfig:
789         cmd = [ "git", "config" ]
790         if typeSpecifier:
791             cmd += [ typeSpecifier ]
792         cmd += [ key ]
793         s = read_pipe(cmd, ignore_error=True)
794         _gitConfig[key] = s.strip()
795     return _gitConfig[key]
796
797 def gitConfigBool(key):
798     """Return a bool, using git config --bool.  It is True only if the
799        variable is set to true, and False if set to false or not present
800        in the config."""
801
802     if key not in _gitConfig:
803         _gitConfig[key] = gitConfig(key, '--bool') == "true"
804     return _gitConfig[key]
805
806 def gitConfigInt(key):
807     if key not in _gitConfig:
808         cmd = [ "git", "config", "--int", key ]
809         s = read_pipe(cmd, ignore_error=True)
810         v = s.strip()
811         try:
812             _gitConfig[key] = int(gitConfig(key, '--int'))
813         except ValueError:
814             _gitConfig[key] = None
815     return _gitConfig[key]
816
817 def gitConfigList(key):
818     if key not in _gitConfig:
819         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
820         _gitConfig[key] = s.strip().splitlines()
821         if _gitConfig[key] == ['']:
822             _gitConfig[key] = []
823     return _gitConfig[key]
824
825 def p4BranchesInGit(branchesAreInRemotes=True):
826     """Find all the branches whose names start with "p4/", looking
827        in remotes or heads as specified by the argument.  Return
828        a dictionary of { branch: revision } for each one found.
829        The branch names are the short names, without any
830        "p4/" prefix."""
831
832     branches = {}
833
834     cmdline = "git rev-parse --symbolic "
835     if branchesAreInRemotes:
836         cmdline += "--remotes"
837     else:
838         cmdline += "--branches"
839
840     for line in read_pipe_lines(cmdline):
841         line = line.strip()
842
843         # only import to p4/
844         if not line.startswith('p4/'):
845             continue
846         # special symbolic ref to p4/master
847         if line == "p4/HEAD":
848             continue
849
850         # strip off p4/ prefix
851         branch = line[len("p4/"):]
852
853         branches[branch] = parseRevision(line)
854
855     return branches
856
857 def branch_exists(branch):
858     """Make sure that the given ref name really exists."""
859
860     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
861     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
862     out, _ = p.communicate()
863     if p.returncode:
864         return False
865     # expect exactly one line of output: the branch name
866     return out.rstrip() == branch
867
868 def findUpstreamBranchPoint(head = "HEAD"):
869     branches = p4BranchesInGit()
870     # map from depot-path to branch name
871     branchByDepotPath = {}
872     for branch in branches.keys():
873         tip = branches[branch]
874         log = extractLogMessageFromGitCommit(tip)
875         settings = extractSettingsGitLog(log)
876         if "depot-paths" in settings:
877             paths = ",".join(settings["depot-paths"])
878             branchByDepotPath[paths] = "remotes/p4/" + branch
879
880     settings = None
881     parent = 0
882     while parent < 65535:
883         commit = head + "~%s" % parent
884         log = extractLogMessageFromGitCommit(commit)
885         settings = extractSettingsGitLog(log)
886         if "depot-paths" in settings:
887             paths = ",".join(settings["depot-paths"])
888             if paths in branchByDepotPath:
889                 return [branchByDepotPath[paths], settings]
890
891         parent = parent + 1
892
893     return ["", settings]
894
895 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
896     if not silent:
897         print("Creating/updating branch(es) in %s based on origin branch(es)"
898                % localRefPrefix)
899
900     originPrefix = "origin/p4/"
901
902     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
903         line = line.strip()
904         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
905             continue
906
907         headName = line[len(originPrefix):]
908         remoteHead = localRefPrefix + headName
909         originHead = line
910
911         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
912         if ('depot-paths' not in original
913             or 'change' not in original):
914             continue
915
916         update = False
917         if not gitBranchExists(remoteHead):
918             if verbose:
919                 print("creating %s" % remoteHead)
920             update = True
921         else:
922             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
923             if 'change' in settings:
924                 if settings['depot-paths'] == original['depot-paths']:
925                     originP4Change = int(original['change'])
926                     p4Change = int(settings['change'])
927                     if originP4Change > p4Change:
928                         print("%s (%s) is newer than %s (%s). "
929                                "Updating p4 branch from origin."
930                                % (originHead, originP4Change,
931                                   remoteHead, p4Change))
932                         update = True
933                 else:
934                     print("Ignoring: %s was imported from %s while "
935                            "%s was imported from %s"
936                            % (originHead, ','.join(original['depot-paths']),
937                               remoteHead, ','.join(settings['depot-paths'])))
938
939         if update:
940             system("git update-ref %s %s" % (remoteHead, originHead))
941
942 def originP4BranchesExist():
943         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
944
945
946 def p4ParseNumericChangeRange(parts):
947     changeStart = int(parts[0][1:])
948     if parts[1] == '#head':
949         changeEnd = p4_last_change()
950     else:
951         changeEnd = int(parts[1])
952
953     return (changeStart, changeEnd)
954
955 def chooseBlockSize(blockSize):
956     if blockSize:
957         return blockSize
958     else:
959         return defaultBlockSize
960
961 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
962     assert depotPaths
963
964     # Parse the change range into start and end. Try to find integer
965     # revision ranges as these can be broken up into blocks to avoid
966     # hitting server-side limits (maxrows, maxscanresults). But if
967     # that doesn't work, fall back to using the raw revision specifier
968     # strings, without using block mode.
969
970     if changeRange is None or changeRange == '':
971         changeStart = 1
972         changeEnd = p4_last_change()
973         block_size = chooseBlockSize(requestedBlockSize)
974     else:
975         parts = changeRange.split(',')
976         assert len(parts) == 2
977         try:
978             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
979             block_size = chooseBlockSize(requestedBlockSize)
980         except ValueError:
981             changeStart = parts[0][1:]
982             changeEnd = parts[1]
983             if requestedBlockSize:
984                 die("cannot use --changes-block-size with non-numeric revisions")
985             block_size = None
986
987     changes = set()
988
989     # Retrieve changes a block at a time, to prevent running
990     # into a MaxResults/MaxScanRows error from the server. If
991     # we _do_ hit one of those errors, turn down the block size
992
993     while True:
994         cmd = ['changes']
995
996         if block_size:
997             end = min(changeEnd, changeStart + block_size)
998             revisionRange = "%d,%d" % (changeStart, end)
999         else:
1000             revisionRange = "%s,%s" % (changeStart, changeEnd)
1001
1002         for p in depotPaths:
1003             cmd += ["%s...@%s" % (p, revisionRange)]
1004
1005         # fetch the changes
1006         try:
1007             result = p4CmdList(cmd, errors_as_exceptions=True)
1008         except P4RequestSizeException as e:
1009             if not block_size:
1010                 block_size = e.limit
1011             elif block_size > e.limit:
1012                 block_size = e.limit
1013             else:
1014                 block_size = max(2, block_size // 2)
1015
1016             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1017             continue
1018         except P4Exception as e:
1019             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1020
1021         # Insert changes in chronological order
1022         for entry in reversed(result):
1023             if 'change' not in entry:
1024                 continue
1025             changes.add(int(entry['change']))
1026
1027         if not block_size:
1028             break
1029
1030         if end >= changeEnd:
1031             break
1032
1033         changeStart = end + 1
1034
1035     changes = sorted(changes)
1036     return changes
1037
1038 def p4PathStartsWith(path, prefix):
1039     # This method tries to remedy a potential mixed-case issue:
1040     #
1041     # If UserA adds  //depot/DirA/file1
1042     # and UserB adds //depot/dira/file2
1043     #
1044     # we may or may not have a problem. If you have core.ignorecase=true,
1045     # we treat DirA and dira as the same directory
1046     if gitConfigBool("core.ignorecase"):
1047         return path.lower().startswith(prefix.lower())
1048     return path.startswith(prefix)
1049
1050 def getClientSpec():
1051     """Look at the p4 client spec, create a View() object that contains
1052        all the mappings, and return it."""
1053
1054     specList = p4CmdList("client -o")
1055     if len(specList) != 1:
1056         die('Output from "client -o" is %d lines, expecting 1' %
1057             len(specList))
1058
1059     # dictionary of all client parameters
1060     entry = specList[0]
1061
1062     # the //client/ name
1063     client_name = entry["Client"]
1064
1065     # just the keys that start with "View"
1066     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1067
1068     # hold this new View
1069     view = View(client_name)
1070
1071     # append the lines, in order, to the view
1072     for view_num in range(len(view_keys)):
1073         k = "View%d" % view_num
1074         if k not in view_keys:
1075             die("Expected view key %s missing" % k)
1076         view.append(entry[k])
1077
1078     return view
1079
1080 def getClientRoot():
1081     """Grab the client directory."""
1082
1083     output = p4CmdList("client -o")
1084     if len(output) != 1:
1085         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1086
1087     entry = output[0]
1088     if "Root" not in entry:
1089         die('Client has no "Root"')
1090
1091     return entry["Root"]
1092
1093 #
1094 # P4 wildcards are not allowed in filenames.  P4 complains
1095 # if you simply add them, but you can force it with "-f", in
1096 # which case it translates them into %xx encoding internally.
1097 #
1098 def wildcard_decode(path):
1099     # Search for and fix just these four characters.  Do % last so
1100     # that fixing it does not inadvertently create new %-escapes.
1101     # Cannot have * in a filename in windows; untested as to
1102     # what p4 would do in such a case.
1103     if not platform.system() == "Windows":
1104         path = path.replace("%2A", "*")
1105     path = path.replace("%23", "#") \
1106                .replace("%40", "@") \
1107                .replace("%25", "%")
1108     return path
1109
1110 def wildcard_encode(path):
1111     # do % first to avoid double-encoding the %s introduced here
1112     path = path.replace("%", "%25") \
1113                .replace("*", "%2A") \
1114                .replace("#", "%23") \
1115                .replace("@", "%40")
1116     return path
1117
1118 def wildcard_present(path):
1119     m = re.search("[*#@%]", path)
1120     return m is not None
1121
1122 class LargeFileSystem(object):
1123     """Base class for large file system support."""
1124
1125     def __init__(self, writeToGitStream):
1126         self.largeFiles = set()
1127         self.writeToGitStream = writeToGitStream
1128
1129     def generatePointer(self, cloneDestination, contentFile):
1130         """Return the content of a pointer file that is stored in Git instead of
1131            the actual content."""
1132         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1133
1134     def pushFile(self, localLargeFile):
1135         """Push the actual content which is not stored in the Git repository to
1136            a server."""
1137         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1138
1139     def hasLargeFileExtension(self, relPath):
1140         return reduce(
1141             lambda a, b: a or b,
1142             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1143             False
1144         )
1145
1146     def generateTempFile(self, contents):
1147         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1148         for d in contents:
1149             contentFile.write(d)
1150         contentFile.close()
1151         return contentFile.name
1152
1153     def exceedsLargeFileThreshold(self, relPath, contents):
1154         if gitConfigInt('git-p4.largeFileThreshold'):
1155             contentsSize = sum(len(d) for d in contents)
1156             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1157                 return True
1158         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159             contentsSize = sum(len(d) for d in contents)
1160             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1161                 return False
1162             contentTempFile = self.generateTempFile(contents)
1163             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1164             with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1165                 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1166                 compressedContentsSize = zf.infolist()[0].compress_size
1167             os.remove(contentTempFile)
1168             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169                 return True
1170         return False
1171
1172     def addLargeFile(self, relPath):
1173         self.largeFiles.add(relPath)
1174
1175     def removeLargeFile(self, relPath):
1176         self.largeFiles.remove(relPath)
1177
1178     def isLargeFile(self, relPath):
1179         return relPath in self.largeFiles
1180
1181     def processContent(self, git_mode, relPath, contents):
1182         """Processes the content of git fast import. This method decides if a
1183            file is stored in the large file system and handles all necessary
1184            steps."""
1185         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186             contentTempFile = self.generateTempFile(contents)
1187             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188             if pointer_git_mode:
1189                 git_mode = pointer_git_mode
1190             if localLargeFile:
1191                 # Move temp file to final location in large file system
1192                 largeFileDir = os.path.dirname(localLargeFile)
1193                 if not os.path.isdir(largeFileDir):
1194                     os.makedirs(largeFileDir)
1195                 shutil.move(contentTempFile, localLargeFile)
1196                 self.addLargeFile(relPath)
1197                 if gitConfigBool('git-p4.largeFilePush'):
1198                     self.pushFile(localLargeFile)
1199                 if verbose:
1200                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201         return (git_mode, contents)
1202
1203 class MockLFS(LargeFileSystem):
1204     """Mock large file system for testing."""
1205
1206     def generatePointer(self, contentFile):
1207         """The pointer content is the original content prefixed with "pointer-".
1208            The local filename of the large file storage is derived from the file content.
1209            """
1210         with open(contentFile, 'r') as f:
1211             content = next(f)
1212             gitMode = '100644'
1213             pointerContents = 'pointer-' + content
1214             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215             return (gitMode, pointerContents, localLargeFile)
1216
1217     def pushFile(self, localLargeFile):
1218         """The remote filename of the large file storage is the same as the local
1219            one but in a different directory.
1220            """
1221         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222         if not os.path.exists(remotePath):
1223             os.makedirs(remotePath)
1224         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1225
1226 class GitLFS(LargeFileSystem):
1227     """Git LFS as backend for the git-p4 large file system.
1228        See https://git-lfs.github.com/ for details."""
1229
1230     def __init__(self, *args):
1231         LargeFileSystem.__init__(self, *args)
1232         self.baseGitAttributes = []
1233
1234     def generatePointer(self, contentFile):
1235         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236            mode and content which is stored in the Git repository instead of
1237            the actual content. Return also the new location of the actual
1238            content.
1239            """
1240         if os.path.getsize(contentFile) == 0:
1241             return (None, '', None)
1242
1243         pointerProcess = subprocess.Popen(
1244             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245             stdout=subprocess.PIPE
1246         )
1247         pointerFile = pointerProcess.stdout.read()
1248         if pointerProcess.wait():
1249             os.remove(contentFile)
1250             die('git-lfs pointer command failed. Did you install the extension?')
1251
1252         # Git LFS removed the preamble in the output of the 'pointer' command
1253         # starting from version 1.2.0. Check for the preamble here to support
1254         # earlier versions.
1255         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256         if pointerFile.startswith('Git LFS pointer for'):
1257             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1258
1259         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260         localLargeFile = os.path.join(
1261             os.getcwd(),
1262             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1263             oid,
1264         )
1265         # LFS Spec states that pointer files should not have the executable bit set.
1266         gitMode = '100644'
1267         return (gitMode, pointerFile, localLargeFile)
1268
1269     def pushFile(self, localLargeFile):
1270         uploadProcess = subprocess.Popen(
1271             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1272         )
1273         if uploadProcess.wait():
1274             die('git-lfs push command failed. Did you define a remote?')
1275
1276     def generateGitAttributes(self):
1277         return (
1278             self.baseGitAttributes +
1279             [
1280                 '\n',
1281                 '#\n',
1282                 '# Git LFS (see https://git-lfs.github.com/)\n',
1283                 '#\n',
1284             ] +
1285             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1286                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1287             ] +
1288             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1289                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1290             ]
1291         )
1292
1293     def addLargeFile(self, relPath):
1294         LargeFileSystem.addLargeFile(self, relPath)
1295         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1296
1297     def removeLargeFile(self, relPath):
1298         LargeFileSystem.removeLargeFile(self, relPath)
1299         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1300
1301     def processContent(self, git_mode, relPath, contents):
1302         if relPath == '.gitattributes':
1303             self.baseGitAttributes = contents
1304             return (git_mode, self.generateGitAttributes())
1305         else:
1306             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1307
1308 class Command:
1309     delete_actions = ( "delete", "move/delete", "purge" )
1310     add_actions = ( "add", "branch", "move/add" )
1311
1312     def __init__(self):
1313         self.usage = "usage: %prog [options]"
1314         self.needsGit = True
1315         self.verbose = False
1316
1317     # This is required for the "append" update_shelve action
1318     def ensure_value(self, attr, value):
1319         if not hasattr(self, attr) or getattr(self, attr) is None:
1320             setattr(self, attr, value)
1321         return getattr(self, attr)
1322
1323 class P4UserMap:
1324     def __init__(self):
1325         self.userMapFromPerforceServer = False
1326         self.myP4UserId = None
1327
1328     def p4UserId(self):
1329         if self.myP4UserId:
1330             return self.myP4UserId
1331
1332         results = p4CmdList("user -o")
1333         for r in results:
1334             if 'User' in r:
1335                 self.myP4UserId = r['User']
1336                 return r['User']
1337         die("Could not find your p4 user id")
1338
1339     def p4UserIsMe(self, p4User):
1340         # return True if the given p4 user is actually me
1341         me = self.p4UserId()
1342         if not p4User or p4User != me:
1343             return False
1344         else:
1345             return True
1346
1347     def getUserCacheFilename(self):
1348         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1349         return home + "/.gitp4-usercache.txt"
1350
1351     def getUserMapFromPerforceServer(self):
1352         if self.userMapFromPerforceServer:
1353             return
1354         self.users = {}
1355         self.emails = {}
1356
1357         for output in p4CmdList("users"):
1358             if "User" not in output:
1359                 continue
1360             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1361             self.emails[output["Email"]] = output["User"]
1362
1363         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1364         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1365             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1366             if mapUser and len(mapUser[0]) == 3:
1367                 user = mapUser[0][0]
1368                 fullname = mapUser[0][1]
1369                 email = mapUser[0][2]
1370                 self.users[user] = fullname + " <" + email + ">"
1371                 self.emails[email] = user
1372
1373         s = ''
1374         for (key, val) in self.users.items():
1375             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1376
1377         open(self.getUserCacheFilename(), "wb").write(s)
1378         self.userMapFromPerforceServer = True
1379
1380     def loadUserMapFromCache(self):
1381         self.users = {}
1382         self.userMapFromPerforceServer = False
1383         try:
1384             cache = open(self.getUserCacheFilename(), "rb")
1385             lines = cache.readlines()
1386             cache.close()
1387             for line in lines:
1388                 entry = line.strip().split("\t")
1389                 self.users[entry[0]] = entry[1]
1390         except IOError:
1391             self.getUserMapFromPerforceServer()
1392
1393 class P4Debug(Command):
1394     def __init__(self):
1395         Command.__init__(self)
1396         self.options = []
1397         self.description = "A tool to debug the output of p4 -G."
1398         self.needsGit = False
1399
1400     def run(self, args):
1401         j = 0
1402         for output in p4CmdList(args):
1403             print('Element: %d' % j)
1404             j += 1
1405             print(output)
1406         return True
1407
1408 class P4RollBack(Command):
1409     def __init__(self):
1410         Command.__init__(self)
1411         self.options = [
1412             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1413         ]
1414         self.description = "A tool to debug the multi-branch import. Don't use :)"
1415         self.rollbackLocalBranches = False
1416
1417     def run(self, args):
1418         if len(args) != 1:
1419             return False
1420         maxChange = int(args[0])
1421
1422         if "p4ExitCode" in p4Cmd("changes -m 1"):
1423             die("Problems executing p4");
1424
1425         if self.rollbackLocalBranches:
1426             refPrefix = "refs/heads/"
1427             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1428         else:
1429             refPrefix = "refs/remotes/"
1430             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1431
1432         for line in lines:
1433             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1434                 line = line.strip()
1435                 ref = refPrefix + line
1436                 log = extractLogMessageFromGitCommit(ref)
1437                 settings = extractSettingsGitLog(log)
1438
1439                 depotPaths = settings['depot-paths']
1440                 change = settings['change']
1441
1442                 changed = False
1443
1444                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1445                                                            for p in depotPaths]))) == 0:
1446                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1447                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1448                     continue
1449
1450                 while change and int(change) > maxChange:
1451                     changed = True
1452                     if self.verbose:
1453                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1454                     system("git update-ref %s \"%s^\"" % (ref, ref))
1455                     log = extractLogMessageFromGitCommit(ref)
1456                     settings =  extractSettingsGitLog(log)
1457
1458
1459                     depotPaths = settings['depot-paths']
1460                     change = settings['change']
1461
1462                 if changed:
1463                     print("%s rewound to %s" % (ref, change))
1464
1465         return True
1466
1467 class P4Submit(Command, P4UserMap):
1468
1469     conflict_behavior_choices = ("ask", "skip", "quit")
1470
1471     def __init__(self):
1472         Command.__init__(self)
1473         P4UserMap.__init__(self)
1474         self.options = [
1475                 optparse.make_option("--origin", dest="origin"),
1476                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1477                 # preserve the user, requires relevant p4 permissions
1478                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1479                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1480                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1481                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1482                 optparse.make_option("--conflict", dest="conflict_behavior",
1483                                      choices=self.conflict_behavior_choices),
1484                 optparse.make_option("--branch", dest="branch"),
1485                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1486                                      help="Shelve instead of submit. Shelved files are reverted, "
1487                                      "restoring the workspace to the state before the shelve"),
1488                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1489                                      metavar="CHANGELIST",
1490                                      help="update an existing shelved changelist, implies --shelve, "
1491                                            "repeat in-order for multiple shelved changelists"),
1492                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1493                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1494                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1495                                      help="Disable rebase after submit is completed. Can be useful if you "
1496                                      "work from a local git branch that is not master"),
1497                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1498                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1499         ]
1500         self.description = """Submit changes from git to the perforce depot.\n
1501     The `p4-pre-submit` hook is executed if it exists and is executable.
1502     The hook takes no parameters and nothing from standard input. Exiting with
1503     non-zero status from this script prevents `git-p4 submit` from launching.
1504
1505     One usage scenario is to run unit tests in the hook."""
1506
1507         self.usage += " [name of git branch to submit into perforce depot]"
1508         self.origin = ""
1509         self.detectRenames = False
1510         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1511         self.dry_run = False
1512         self.shelve = False
1513         self.update_shelve = list()
1514         self.commit = ""
1515         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1516         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1517         self.prepare_p4_only = False
1518         self.conflict_behavior = None
1519         self.isWindows = (platform.system() == "Windows")
1520         self.exportLabels = False
1521         self.p4HasMoveCommand = p4_has_move_command()
1522         self.branch = None
1523
1524         if gitConfig('git-p4.largeFileSystem'):
1525             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1526
1527     def check(self):
1528         if len(p4CmdList("opened ...")) > 0:
1529             die("You have files opened with perforce! Close them before starting the sync.")
1530
1531     def separate_jobs_from_description(self, message):
1532         """Extract and return a possible Jobs field in the commit
1533            message.  It goes into a separate section in the p4 change
1534            specification.
1535
1536            A jobs line starts with "Jobs:" and looks like a new field
1537            in a form.  Values are white-space separated on the same
1538            line or on following lines that start with a tab.
1539
1540            This does not parse and extract the full git commit message
1541            like a p4 form.  It just sees the Jobs: line as a marker
1542            to pass everything from then on directly into the p4 form,
1543            but outside the description section.
1544
1545            Return a tuple (stripped log message, jobs string)."""
1546
1547         m = re.search(r'^Jobs:', message, re.MULTILINE)
1548         if m is None:
1549             return (message, None)
1550
1551         jobtext = message[m.start():]
1552         stripped_message = message[:m.start()].rstrip()
1553         return (stripped_message, jobtext)
1554
1555     def prepareLogMessage(self, template, message, jobs):
1556         """Edits the template returned from "p4 change -o" to insert
1557            the message in the Description field, and the jobs text in
1558            the Jobs field."""
1559         result = ""
1560
1561         inDescriptionSection = False
1562
1563         for line in template.split("\n"):
1564             if line.startswith("#"):
1565                 result += line + "\n"
1566                 continue
1567
1568             if inDescriptionSection:
1569                 if line.startswith("Files:") or line.startswith("Jobs:"):
1570                     inDescriptionSection = False
1571                     # insert Jobs section
1572                     if jobs:
1573                         result += jobs + "\n"
1574                 else:
1575                     continue
1576             else:
1577                 if line.startswith("Description:"):
1578                     inDescriptionSection = True
1579                     line += "\n"
1580                     for messageLine in message.split("\n"):
1581                         line += "\t" + messageLine + "\n"
1582
1583             result += line + "\n"
1584
1585         return result
1586
1587     def patchRCSKeywords(self, file, pattern):
1588         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1589         (handle, outFileName) = tempfile.mkstemp(dir='.')
1590         try:
1591             outFile = os.fdopen(handle, "w+")
1592             inFile = open(file, "r")
1593             regexp = re.compile(pattern, re.VERBOSE)
1594             for line in inFile.readlines():
1595                 line = regexp.sub(r'$\1$', line)
1596                 outFile.write(line)
1597             inFile.close()
1598             outFile.close()
1599             # Forcibly overwrite the original file
1600             os.unlink(file)
1601             shutil.move(outFileName, file)
1602         except:
1603             # cleanup our temporary file
1604             os.unlink(outFileName)
1605             print("Failed to strip RCS keywords in %s" % file)
1606             raise
1607
1608         print("Patched up RCS keywords in %s" % file)
1609
1610     def p4UserForCommit(self,id):
1611         # Return the tuple (perforce user,git email) for a given git commit id
1612         self.getUserMapFromPerforceServer()
1613         gitEmail = read_pipe(["git", "log", "--max-count=1",
1614                               "--format=%ae", id])
1615         gitEmail = gitEmail.strip()
1616         if gitEmail not in self.emails:
1617             return (None,gitEmail)
1618         else:
1619             return (self.emails[gitEmail],gitEmail)
1620
1621     def checkValidP4Users(self,commits):
1622         # check if any git authors cannot be mapped to p4 users
1623         for id in commits:
1624             (user,email) = self.p4UserForCommit(id)
1625             if not user:
1626                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1627                 if gitConfigBool("git-p4.allowMissingP4Users"):
1628                     print("%s" % msg)
1629                 else:
1630                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1631
1632     def lastP4Changelist(self):
1633         # Get back the last changelist number submitted in this client spec. This
1634         # then gets used to patch up the username in the change. If the same
1635         # client spec is being used by multiple processes then this might go
1636         # wrong.
1637         results = p4CmdList("client -o")        # find the current client
1638         client = None
1639         for r in results:
1640             if 'Client' in r:
1641                 client = r['Client']
1642                 break
1643         if not client:
1644             die("could not get client spec")
1645         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1646         for r in results:
1647             if 'change' in r:
1648                 return r['change']
1649         die("Could not get changelist number for last submit - cannot patch up user details")
1650
1651     def modifyChangelistUser(self, changelist, newUser):
1652         # fixup the user field of a changelist after it has been submitted.
1653         changes = p4CmdList("change -o %s" % changelist)
1654         if len(changes) != 1:
1655             die("Bad output from p4 change modifying %s to user %s" %
1656                 (changelist, newUser))
1657
1658         c = changes[0]
1659         if c['User'] == newUser: return   # nothing to do
1660         c['User'] = newUser
1661         input = marshal.dumps(c)
1662
1663         result = p4CmdList("change -f -i", stdin=input)
1664         for r in result:
1665             if 'code' in r:
1666                 if r['code'] == 'error':
1667                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1668             if 'data' in r:
1669                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1670                 return
1671         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1672
1673     def canChangeChangelists(self):
1674         # check to see if we have p4 admin or super-user permissions, either of
1675         # which are required to modify changelists.
1676         results = p4CmdList(["protects", self.depotPath])
1677         for r in results:
1678             if 'perm' in r:
1679                 if r['perm'] == 'admin':
1680                     return 1
1681                 if r['perm'] == 'super':
1682                     return 1
1683         return 0
1684
1685     def prepareSubmitTemplate(self, changelist=None):
1686         """Run "p4 change -o" to grab a change specification template.
1687            This does not use "p4 -G", as it is nice to keep the submission
1688            template in original order, since a human might edit it.
1689
1690            Remove lines in the Files section that show changes to files
1691            outside the depot path we're committing into."""
1692
1693         [upstream, settings] = findUpstreamBranchPoint()
1694
1695         template = """\
1696 # A Perforce Change Specification.
1697 #
1698 #  Change:      The change number. 'new' on a new changelist.
1699 #  Date:        The date this specification was last modified.
1700 #  Client:      The client on which the changelist was created.  Read-only.
1701 #  User:        The user who created the changelist.
1702 #  Status:      Either 'pending' or 'submitted'. Read-only.
1703 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1704 #  Description: Comments about the changelist.  Required.
1705 #  Jobs:        What opened jobs are to be closed by this changelist.
1706 #               You may delete jobs from this list.  (New changelists only.)
1707 #  Files:       What opened files from the default changelist are to be added
1708 #               to this changelist.  You may delete files from this list.
1709 #               (New changelists only.)
1710 """
1711         files_list = []
1712         inFilesSection = False
1713         change_entry = None
1714         args = ['change', '-o']
1715         if changelist:
1716             args.append(str(changelist))
1717         for entry in p4CmdList(args):
1718             if 'code' not in entry:
1719                 continue
1720             if entry['code'] == 'stat':
1721                 change_entry = entry
1722                 break
1723         if not change_entry:
1724             die('Failed to decode output of p4 change -o')
1725         for key, value in change_entry.iteritems():
1726             if key.startswith('File'):
1727                 if 'depot-paths' in settings:
1728                     if not [p for p in settings['depot-paths']
1729                             if p4PathStartsWith(value, p)]:
1730                         continue
1731                 else:
1732                     if not p4PathStartsWith(value, self.depotPath):
1733                         continue
1734                 files_list.append(value)
1735                 continue
1736         # Output in the order expected by prepareLogMessage
1737         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1738             if key not in change_entry:
1739                 continue
1740             template += '\n'
1741             template += key + ':'
1742             if key == 'Description':
1743                 template += '\n'
1744             for field_line in change_entry[key].splitlines():
1745                 template += '\t'+field_line+'\n'
1746         if len(files_list) > 0:
1747             template += '\n'
1748             template += 'Files:\n'
1749         for path in files_list:
1750             template += '\t'+path+'\n'
1751         return template
1752
1753     def edit_template(self, template_file):
1754         """Invoke the editor to let the user change the submission
1755            message.  Return true if okay to continue with the submit."""
1756
1757         # if configured to skip the editing part, just submit
1758         if gitConfigBool("git-p4.skipSubmitEdit"):
1759             return True
1760
1761         # look at the modification time, to check later if the user saved
1762         # the file
1763         mtime = os.stat(template_file).st_mtime
1764
1765         # invoke the editor
1766         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1767             editor = os.environ.get("P4EDITOR")
1768         else:
1769             editor = read_pipe("git var GIT_EDITOR").strip()
1770         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1771
1772         # If the file was not saved, prompt to see if this patch should
1773         # be skipped.  But skip this verification step if configured so.
1774         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1775             return True
1776
1777         # modification time updated means user saved the file
1778         if os.stat(template_file).st_mtime > mtime:
1779             return True
1780
1781         while True:
1782             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1783             if response == 'y':
1784                 return True
1785             if response == 'n':
1786                 return False
1787
1788     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1789         # diff
1790         if "P4DIFF" in os.environ:
1791             del(os.environ["P4DIFF"])
1792         diff = ""
1793         for editedFile in editedFiles:
1794             diff += p4_read_pipe(['diff', '-du',
1795                                   wildcard_encode(editedFile)])
1796
1797         # new file diff
1798         newdiff = ""
1799         for newFile in filesToAdd:
1800             newdiff += "==== new file ====\n"
1801             newdiff += "--- /dev/null\n"
1802             newdiff += "+++ %s\n" % newFile
1803
1804             is_link = os.path.islink(newFile)
1805             expect_link = newFile in symlinks
1806
1807             if is_link and expect_link:
1808                 newdiff += "+%s\n" % os.readlink(newFile)
1809             else:
1810                 f = open(newFile, "r")
1811                 for line in f.readlines():
1812                     newdiff += "+" + line
1813                 f.close()
1814
1815         return (diff + newdiff).replace('\r\n', '\n')
1816
1817     def applyCommit(self, id):
1818         """Apply one commit, return True if it succeeded."""
1819
1820         print("Applying", read_pipe(["git", "show", "-s",
1821                                      "--format=format:%h %s", id]))
1822
1823         (p4User, gitEmail) = self.p4UserForCommit(id)
1824
1825         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1826         filesToAdd = set()
1827         filesToChangeType = set()
1828         filesToDelete = set()
1829         editedFiles = set()
1830         pureRenameCopy = set()
1831         symlinks = set()
1832         filesToChangeExecBit = {}
1833         all_files = list()
1834
1835         for line in diff:
1836             diff = parseDiffTreeEntry(line)
1837             modifier = diff['status']
1838             path = diff['src']
1839             all_files.append(path)
1840
1841             if modifier == "M":
1842                 p4_edit(path)
1843                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1844                     filesToChangeExecBit[path] = diff['dst_mode']
1845                 editedFiles.add(path)
1846             elif modifier == "A":
1847                 filesToAdd.add(path)
1848                 filesToChangeExecBit[path] = diff['dst_mode']
1849                 if path in filesToDelete:
1850                     filesToDelete.remove(path)
1851
1852                 dst_mode = int(diff['dst_mode'], 8)
1853                 if dst_mode == 0o120000:
1854                     symlinks.add(path)
1855
1856             elif modifier == "D":
1857                 filesToDelete.add(path)
1858                 if path in filesToAdd:
1859                     filesToAdd.remove(path)
1860             elif modifier == "C":
1861                 src, dest = diff['src'], diff['dst']
1862                 all_files.append(dest)
1863                 p4_integrate(src, dest)
1864                 pureRenameCopy.add(dest)
1865                 if diff['src_sha1'] != diff['dst_sha1']:
1866                     p4_edit(dest)
1867                     pureRenameCopy.discard(dest)
1868                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1869                     p4_edit(dest)
1870                     pureRenameCopy.discard(dest)
1871                     filesToChangeExecBit[dest] = diff['dst_mode']
1872                 if self.isWindows:
1873                     # turn off read-only attribute
1874                     os.chmod(dest, stat.S_IWRITE)
1875                 os.unlink(dest)
1876                 editedFiles.add(dest)
1877             elif modifier == "R":
1878                 src, dest = diff['src'], diff['dst']
1879                 all_files.append(dest)
1880                 if self.p4HasMoveCommand:
1881                     p4_edit(src)        # src must be open before move
1882                     p4_move(src, dest)  # opens for (move/delete, move/add)
1883                 else:
1884                     p4_integrate(src, dest)
1885                     if diff['src_sha1'] != diff['dst_sha1']:
1886                         p4_edit(dest)
1887                     else:
1888                         pureRenameCopy.add(dest)
1889                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1890                     if not self.p4HasMoveCommand:
1891                         p4_edit(dest)   # with move: already open, writable
1892                     filesToChangeExecBit[dest] = diff['dst_mode']
1893                 if not self.p4HasMoveCommand:
1894                     if self.isWindows:
1895                         os.chmod(dest, stat.S_IWRITE)
1896                     os.unlink(dest)
1897                     filesToDelete.add(src)
1898                 editedFiles.add(dest)
1899             elif modifier == "T":
1900                 filesToChangeType.add(path)
1901             else:
1902                 die("unknown modifier %s for %s" % (modifier, path))
1903
1904         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1905         patchcmd = diffcmd + " | git apply "
1906         tryPatchCmd = patchcmd + "--check -"
1907         applyPatchCmd = patchcmd + "--check --apply -"
1908         patch_succeeded = True
1909
1910         if os.system(tryPatchCmd) != 0:
1911             fixed_rcs_keywords = False
1912             patch_succeeded = False
1913             print("Unfortunately applying the change failed!")
1914
1915             # Patch failed, maybe it's just RCS keyword woes. Look through
1916             # the patch to see if that's possible.
1917             if gitConfigBool("git-p4.attemptRCSCleanup"):
1918                 file = None
1919                 pattern = None
1920                 kwfiles = {}
1921                 for file in editedFiles | filesToDelete:
1922                     # did this file's delta contain RCS keywords?
1923                     pattern = p4_keywords_regexp_for_file(file)
1924
1925                     if pattern:
1926                         # this file is a possibility...look for RCS keywords.
1927                         regexp = re.compile(pattern, re.VERBOSE)
1928                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1929                             if regexp.search(line):
1930                                 if verbose:
1931                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1932                                 kwfiles[file] = pattern
1933                                 break
1934
1935                 for file in kwfiles:
1936                     if verbose:
1937                         print("zapping %s with %s" % (line,pattern))
1938                     # File is being deleted, so not open in p4.  Must
1939                     # disable the read-only bit on windows.
1940                     if self.isWindows and file not in editedFiles:
1941                         os.chmod(file, stat.S_IWRITE)
1942                     self.patchRCSKeywords(file, kwfiles[file])
1943                     fixed_rcs_keywords = True
1944
1945             if fixed_rcs_keywords:
1946                 print("Retrying the patch with RCS keywords cleaned up")
1947                 if os.system(tryPatchCmd) == 0:
1948                     patch_succeeded = True
1949
1950         if not patch_succeeded:
1951             for f in editedFiles:
1952                 p4_revert(f)
1953             return False
1954
1955         #
1956         # Apply the patch for real, and do add/delete/+x handling.
1957         #
1958         system(applyPatchCmd)
1959
1960         for f in filesToChangeType:
1961             p4_edit(f, "-t", "auto")
1962         for f in filesToAdd:
1963             p4_add(f)
1964         for f in filesToDelete:
1965             p4_revert(f)
1966             p4_delete(f)
1967
1968         # Set/clear executable bits
1969         for f in filesToChangeExecBit.keys():
1970             mode = filesToChangeExecBit[f]
1971             setP4ExecBit(f, mode)
1972
1973         update_shelve = 0
1974         if len(self.update_shelve) > 0:
1975             update_shelve = self.update_shelve.pop(0)
1976             p4_reopen_in_change(update_shelve, all_files)
1977
1978         #
1979         # Build p4 change description, starting with the contents
1980         # of the git commit message.
1981         #
1982         logMessage = extractLogMessageFromGitCommit(id)
1983         logMessage = logMessage.strip()
1984         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1985
1986         template = self.prepareSubmitTemplate(update_shelve)
1987         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1988
1989         if self.preserveUser:
1990            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1991
1992         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1993             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1994             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1995             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1996
1997         separatorLine = "######## everything below this line is just the diff #######\n"
1998         if not self.prepare_p4_only:
1999             submitTemplate += separatorLine
2000             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2001
2002         (handle, fileName) = tempfile.mkstemp()
2003         tmpFile = os.fdopen(handle, "w+b")
2004         if self.isWindows:
2005             submitTemplate = submitTemplate.replace("\n", "\r\n")
2006         tmpFile.write(submitTemplate)
2007         tmpFile.close()
2008
2009         if self.prepare_p4_only:
2010             #
2011             # Leave the p4 tree prepared, and the submit template around
2012             # and let the user decide what to do next
2013             #
2014             print()
2015             print("P4 workspace prepared for submission.")
2016             print("To submit or revert, go to client workspace")
2017             print("  " + self.clientPath)
2018             print()
2019             print("To submit, use \"p4 submit\" to write a new description,")
2020             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2021                   " \"git p4\"." % fileName)
2022             print("You can delete the file \"%s\" when finished." % fileName)
2023
2024             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2025                 print("To preserve change ownership by user %s, you must\n" \
2026                       "do \"p4 change -f <change>\" after submitting and\n" \
2027                       "edit the User field.")
2028             if pureRenameCopy:
2029                 print("After submitting, renamed files must be re-synced.")
2030                 print("Invoke \"p4 sync -f\" on each of these files:")
2031                 for f in pureRenameCopy:
2032                     print("  " + f)
2033
2034             print()
2035             print("To revert the changes, use \"p4 revert ...\", and delete")
2036             print("the submit template file \"%s\"" % fileName)
2037             if filesToAdd:
2038                 print("Since the commit adds new files, they must be deleted:")
2039                 for f in filesToAdd:
2040                     print("  " + f)
2041             print()
2042             return True
2043
2044         #
2045         # Let the user edit the change description, then submit it.
2046         #
2047         submitted = False
2048
2049         try:
2050             if self.edit_template(fileName):
2051                 # read the edited message and submit
2052                 tmpFile = open(fileName, "rb")
2053                 message = tmpFile.read()
2054                 tmpFile.close()
2055                 if self.isWindows:
2056                     message = message.replace("\r\n", "\n")
2057                 submitTemplate = message[:message.index(separatorLine)]
2058
2059                 if update_shelve:
2060                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2061                 elif self.shelve:
2062                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2063                 else:
2064                     p4_write_pipe(['submit', '-i'], submitTemplate)
2065                     # The rename/copy happened by applying a patch that created a
2066                     # new file.  This leaves it writable, which confuses p4.
2067                     for f in pureRenameCopy:
2068                         p4_sync(f, "-f")
2069
2070                 if self.preserveUser:
2071                     if p4User:
2072                         # Get last changelist number. Cannot easily get it from
2073                         # the submit command output as the output is
2074                         # unmarshalled.
2075                         changelist = self.lastP4Changelist()
2076                         self.modifyChangelistUser(changelist, p4User)
2077
2078                 submitted = True
2079
2080         finally:
2081             # skip this patch
2082             if not submitted or self.shelve:
2083                 if self.shelve:
2084                     print ("Reverting shelved files.")
2085                 else:
2086                     print ("Submission cancelled, undoing p4 changes.")
2087                 for f in editedFiles | filesToDelete:
2088                     p4_revert(f)
2089                 for f in filesToAdd:
2090                     p4_revert(f)
2091                     os.remove(f)
2092
2093         os.remove(fileName)
2094         return submitted
2095
2096     # Export git tags as p4 labels. Create a p4 label and then tag
2097     # with that.
2098     def exportGitTags(self, gitTags):
2099         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2100         if len(validLabelRegexp) == 0:
2101             validLabelRegexp = defaultLabelRegexp
2102         m = re.compile(validLabelRegexp)
2103
2104         for name in gitTags:
2105
2106             if not m.match(name):
2107                 if verbose:
2108                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2109                 continue
2110
2111             # Get the p4 commit this corresponds to
2112             logMessage = extractLogMessageFromGitCommit(name)
2113             values = extractSettingsGitLog(logMessage)
2114
2115             if 'change' not in values:
2116                 # a tag pointing to something not sent to p4; ignore
2117                 if verbose:
2118                     print("git tag %s does not give a p4 commit" % name)
2119                 continue
2120             else:
2121                 changelist = values['change']
2122
2123             # Get the tag details.
2124             inHeader = True
2125             isAnnotated = False
2126             body = []
2127             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2128                 l = l.strip()
2129                 if inHeader:
2130                     if re.match(r'tag\s+', l):
2131                         isAnnotated = True
2132                     elif re.match(r'\s*$', l):
2133                         inHeader = False
2134                         continue
2135                 else:
2136                     body.append(l)
2137
2138             if not isAnnotated:
2139                 body = ["lightweight tag imported by git p4\n"]
2140
2141             # Create the label - use the same view as the client spec we are using
2142             clientSpec = getClientSpec()
2143
2144             labelTemplate  = "Label: %s\n" % name
2145             labelTemplate += "Description:\n"
2146             for b in body:
2147                 labelTemplate += "\t" + b + "\n"
2148             labelTemplate += "View:\n"
2149             for depot_side in clientSpec.mappings:
2150                 labelTemplate += "\t%s\n" % depot_side
2151
2152             if self.dry_run:
2153                 print("Would create p4 label %s for tag" % name)
2154             elif self.prepare_p4_only:
2155                 print("Not creating p4 label %s for tag due to option" \
2156                       " --prepare-p4-only" % name)
2157             else:
2158                 p4_write_pipe(["label", "-i"], labelTemplate)
2159
2160                 # Use the label
2161                 p4_system(["tag", "-l", name] +
2162                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2163
2164                 if verbose:
2165                     print("created p4 label for tag %s" % name)
2166
2167     def run(self, args):
2168         if len(args) == 0:
2169             self.master = currentGitBranch()
2170         elif len(args) == 1:
2171             self.master = args[0]
2172             if not branchExists(self.master):
2173                 die("Branch %s does not exist" % self.master)
2174         else:
2175             return False
2176
2177         for i in self.update_shelve:
2178             if i <= 0:
2179                 sys.exit("invalid changelist %d" % i)
2180
2181         if self.master:
2182             allowSubmit = gitConfig("git-p4.allowSubmit")
2183             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2184                 die("%s is not in git-p4.allowSubmit" % self.master)
2185
2186         [upstream, settings] = findUpstreamBranchPoint()
2187         self.depotPath = settings['depot-paths'][0]
2188         if len(self.origin) == 0:
2189             self.origin = upstream
2190
2191         if len(self.update_shelve) > 0:
2192             self.shelve = True
2193
2194         if self.preserveUser:
2195             if not self.canChangeChangelists():
2196                 die("Cannot preserve user names without p4 super-user or admin permissions")
2197
2198         # if not set from the command line, try the config file
2199         if self.conflict_behavior is None:
2200             val = gitConfig("git-p4.conflict")
2201             if val:
2202                 if val not in self.conflict_behavior_choices:
2203                     die("Invalid value '%s' for config git-p4.conflict" % val)
2204             else:
2205                 val = "ask"
2206             self.conflict_behavior = val
2207
2208         if self.verbose:
2209             print("Origin branch is " + self.origin)
2210
2211         if len(self.depotPath) == 0:
2212             print("Internal error: cannot locate perforce depot path from existing branches")
2213             sys.exit(128)
2214
2215         self.useClientSpec = False
2216         if gitConfigBool("git-p4.useclientspec"):
2217             self.useClientSpec = True
2218         if self.useClientSpec:
2219             self.clientSpecDirs = getClientSpec()
2220
2221         # Check for the existence of P4 branches
2222         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2223
2224         if self.useClientSpec and not branchesDetected:
2225             # all files are relative to the client spec
2226             self.clientPath = getClientRoot()
2227         else:
2228             self.clientPath = p4Where(self.depotPath)
2229
2230         if self.clientPath == "":
2231             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2232
2233         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2234         self.oldWorkingDirectory = os.getcwd()
2235
2236         # ensure the clientPath exists
2237         new_client_dir = False
2238         if not os.path.exists(self.clientPath):
2239             new_client_dir = True
2240             os.makedirs(self.clientPath)
2241
2242         chdir(self.clientPath, is_client_path=True)
2243         if self.dry_run:
2244             print("Would synchronize p4 checkout in %s" % self.clientPath)
2245         else:
2246             print("Synchronizing p4 checkout...")
2247             if new_client_dir:
2248                 # old one was destroyed, and maybe nobody told p4
2249                 p4_sync("...", "-f")
2250             else:
2251                 p4_sync("...")
2252         self.check()
2253
2254         commits = []
2255         if self.master:
2256             committish = self.master
2257         else:
2258             committish = 'HEAD'
2259
2260         if self.commit != "":
2261             if self.commit.find("..") != -1:
2262                 limits_ish = self.commit.split("..")
2263                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2264                     commits.append(line.strip())
2265                 commits.reverse()
2266             else:
2267                 commits.append(self.commit)
2268         else:
2269             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2270                 commits.append(line.strip())
2271             commits.reverse()
2272
2273         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2274             self.checkAuthorship = False
2275         else:
2276             self.checkAuthorship = True
2277
2278         if self.preserveUser:
2279             self.checkValidP4Users(commits)
2280
2281         #
2282         # Build up a set of options to be passed to diff when
2283         # submitting each commit to p4.
2284         #
2285         if self.detectRenames:
2286             # command-line -M arg
2287             self.diffOpts = "-M"
2288         else:
2289             # If not explicitly set check the config variable
2290             detectRenames = gitConfig("git-p4.detectRenames")
2291
2292             if detectRenames.lower() == "false" or detectRenames == "":
2293                 self.diffOpts = ""
2294             elif detectRenames.lower() == "true":
2295                 self.diffOpts = "-M"
2296             else:
2297                 self.diffOpts = "-M%s" % detectRenames
2298
2299         # no command-line arg for -C or --find-copies-harder, just
2300         # config variables
2301         detectCopies = gitConfig("git-p4.detectCopies")
2302         if detectCopies.lower() == "false" or detectCopies == "":
2303             pass
2304         elif detectCopies.lower() == "true":
2305             self.diffOpts += " -C"
2306         else:
2307             self.diffOpts += " -C%s" % detectCopies
2308
2309         if gitConfigBool("git-p4.detectCopiesHarder"):
2310             self.diffOpts += " --find-copies-harder"
2311
2312         num_shelves = len(self.update_shelve)
2313         if num_shelves > 0 and num_shelves != len(commits):
2314             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2315                      (len(commits), num_shelves))
2316
2317         hooks_path = gitConfig("core.hooksPath")
2318         if len(hooks_path) <= 0:
2319             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2320
2321         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2322         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2323             sys.exit(1)
2324
2325         #
2326         # Apply the commits, one at a time.  On failure, ask if should
2327         # continue to try the rest of the patches, or quit.
2328         #
2329         if self.dry_run:
2330             print("Would apply")
2331         applied = []
2332         last = len(commits) - 1
2333         for i, commit in enumerate(commits):
2334             if self.dry_run:
2335                 print(" ", read_pipe(["git", "show", "-s",
2336                                       "--format=format:%h %s", commit]))
2337                 ok = True
2338             else:
2339                 ok = self.applyCommit(commit)
2340             if ok:
2341                 applied.append(commit)
2342             else:
2343                 if self.prepare_p4_only and i < last:
2344                     print("Processing only the first commit due to option" \
2345                           " --prepare-p4-only")
2346                     break
2347                 if i < last:
2348                     quit = False
2349                     while True:
2350                         # prompt for what to do, or use the option/variable
2351                         if self.conflict_behavior == "ask":
2352                             print("What do you want to do?")
2353                             response = raw_input("[s]kip this commit but apply"
2354                                                  " the rest, or [q]uit? ")
2355                             if not response:
2356                                 continue
2357                         elif self.conflict_behavior == "skip":
2358                             response = "s"
2359                         elif self.conflict_behavior == "quit":
2360                             response = "q"
2361                         else:
2362                             die("Unknown conflict_behavior '%s'" %
2363                                 self.conflict_behavior)
2364
2365                         if response[0] == "s":
2366                             print("Skipping this commit, but applying the rest")
2367                             break
2368                         if response[0] == "q":
2369                             print("Quitting")
2370                             quit = True
2371                             break
2372                     if quit:
2373                         break
2374
2375         chdir(self.oldWorkingDirectory)
2376         shelved_applied = "shelved" if self.shelve else "applied"
2377         if self.dry_run:
2378             pass
2379         elif self.prepare_p4_only:
2380             pass
2381         elif len(commits) == len(applied):
2382             print("All commits {0}!".format(shelved_applied))
2383
2384             sync = P4Sync()
2385             if self.branch:
2386                 sync.branch = self.branch
2387             if self.disable_p4sync:
2388                 sync.sync_origin_only()
2389             else:
2390                 sync.run([])
2391
2392                 if not self.disable_rebase:
2393                     rebase = P4Rebase()
2394                     rebase.rebase()
2395
2396         else:
2397             if len(applied) == 0:
2398                 print("No commits {0}.".format(shelved_applied))
2399             else:
2400                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2401                 for c in commits:
2402                     if c in applied:
2403                         star = "*"
2404                     else:
2405                         star = " "
2406                     print(star, read_pipe(["git", "show", "-s",
2407                                            "--format=format:%h %s",  c]))
2408                 print("You will have to do 'git p4 sync' and rebase.")
2409
2410         if gitConfigBool("git-p4.exportLabels"):
2411             self.exportLabels = True
2412
2413         if self.exportLabels:
2414             p4Labels = getP4Labels(self.depotPath)
2415             gitTags = getGitTags()
2416
2417             missingGitTags = gitTags - p4Labels
2418             self.exportGitTags(missingGitTags)
2419
2420         # exit with error unless everything applied perfectly
2421         if len(commits) != len(applied):
2422                 sys.exit(1)
2423
2424         return True
2425
2426 class View(object):
2427     """Represent a p4 view ("p4 help views"), and map files in a
2428        repo according to the view."""
2429
2430     def __init__(self, client_name):
2431         self.mappings = []
2432         self.client_prefix = "//%s/" % client_name
2433         # cache results of "p4 where" to lookup client file locations
2434         self.client_spec_path_cache = {}
2435
2436     def append(self, view_line):
2437         """Parse a view line, splitting it into depot and client
2438            sides.  Append to self.mappings, preserving order.  This
2439            is only needed for tag creation."""
2440
2441         # Split the view line into exactly two words.  P4 enforces
2442         # structure on these lines that simplifies this quite a bit.
2443         #
2444         # Either or both words may be double-quoted.
2445         # Single quotes do not matter.
2446         # Double-quote marks cannot occur inside the words.
2447         # A + or - prefix is also inside the quotes.
2448         # There are no quotes unless they contain a space.
2449         # The line is already white-space stripped.
2450         # The two words are separated by a single space.
2451         #
2452         if view_line[0] == '"':
2453             # First word is double quoted.  Find its end.
2454             close_quote_index = view_line.find('"', 1)
2455             if close_quote_index <= 0:
2456                 die("No first-word closing quote found: %s" % view_line)
2457             depot_side = view_line[1:close_quote_index]
2458             # skip closing quote and space
2459             rhs_index = close_quote_index + 1 + 1
2460         else:
2461             space_index = view_line.find(" ")
2462             if space_index <= 0:
2463                 die("No word-splitting space found: %s" % view_line)
2464             depot_side = view_line[0:space_index]
2465             rhs_index = space_index + 1
2466
2467         # prefix + means overlay on previous mapping
2468         if depot_side.startswith("+"):
2469             depot_side = depot_side[1:]
2470
2471         # prefix - means exclude this path, leave out of mappings
2472         exclude = False
2473         if depot_side.startswith("-"):
2474             exclude = True
2475             depot_side = depot_side[1:]
2476
2477         if not exclude:
2478             self.mappings.append(depot_side)
2479
2480     def convert_client_path(self, clientFile):
2481         # chop off //client/ part to make it relative
2482         if not clientFile.startswith(self.client_prefix):
2483             die("No prefix '%s' on clientFile '%s'" %
2484                 (self.client_prefix, clientFile))
2485         return clientFile[len(self.client_prefix):]
2486
2487     def update_client_spec_path_cache(self, files):
2488         """ Caching file paths by "p4 where" batch query """
2489
2490         # List depot file paths exclude that already cached
2491         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2492
2493         if len(fileArgs) == 0:
2494             return  # All files in cache
2495
2496         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2497         for res in where_result:
2498             if "code" in res and res["code"] == "error":
2499                 # assume error is "... file(s) not in client view"
2500                 continue
2501             if "clientFile" not in res:
2502                 die("No clientFile in 'p4 where' output")
2503             if "unmap" in res:
2504                 # it will list all of them, but only one not unmap-ped
2505                 continue
2506             if gitConfigBool("core.ignorecase"):
2507                 res['depotFile'] = res['depotFile'].lower()
2508             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2509
2510         # not found files or unmap files set to ""
2511         for depotFile in fileArgs:
2512             if gitConfigBool("core.ignorecase"):
2513                 depotFile = depotFile.lower()
2514             if depotFile not in self.client_spec_path_cache:
2515                 self.client_spec_path_cache[depotFile] = ""
2516
2517     def map_in_client(self, depot_path):
2518         """Return the relative location in the client where this
2519            depot file should live.  Returns "" if the file should
2520            not be mapped in the client."""
2521
2522         if gitConfigBool("core.ignorecase"):
2523             depot_path = depot_path.lower()
2524
2525         if depot_path in self.client_spec_path_cache:
2526             return self.client_spec_path_cache[depot_path]
2527
2528         die( "Error: %s is not found in client spec path" % depot_path )
2529         return ""
2530
2531 def cloneExcludeCallback(option, opt_str, value, parser):
2532     # prepend "/" because the first "/" was consumed as part of the option itself.
2533     # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2534     parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2535
2536 class P4Sync(Command, P4UserMap):
2537
2538     def __init__(self):
2539         Command.__init__(self)
2540         P4UserMap.__init__(self)
2541         self.options = [
2542                 optparse.make_option("--branch", dest="branch"),
2543                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2544                 optparse.make_option("--changesfile", dest="changesFile"),
2545                 optparse.make_option("--silent", dest="silent", action="store_true"),
2546                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2547                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2548                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2549                                      help="Import into refs/heads/ , not refs/remotes"),
2550                 optparse.make_option("--max-changes", dest="maxChanges",
2551                                      help="Maximum number of changes to import"),
2552                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2553                                      help="Internal block size to use when iteratively calling p4 changes"),
2554                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2555                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2556                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2557                                      help="Only sync files that are included in the Perforce Client Spec"),
2558                 optparse.make_option("-/", dest="cloneExclude",
2559                                      action="callback", callback=cloneExcludeCallback, type="string",
2560                                      help="exclude depot path"),
2561         ]
2562         self.description = """Imports from Perforce into a git repository.\n
2563     example:
2564     //depot/my/project/ -- to import the current head
2565     //depot/my/project/@all -- to import everything
2566     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2567
2568     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2569
2570         self.usage += " //depot/path[@revRange]"
2571         self.silent = False
2572         self.createdBranches = set()
2573         self.committedChanges = set()
2574         self.branch = ""
2575         self.detectBranches = False
2576         self.detectLabels = False
2577         self.importLabels = False
2578         self.changesFile = ""
2579         self.syncWithOrigin = True
2580         self.importIntoRemotes = True
2581         self.maxChanges = ""
2582         self.changes_block_size = None
2583         self.keepRepoPath = False
2584         self.depotPaths = None
2585         self.p4BranchesInGit = []
2586         self.cloneExclude = []
2587         self.useClientSpec = False
2588         self.useClientSpec_from_options = False
2589         self.clientSpecDirs = None
2590         self.tempBranches = []
2591         self.tempBranchLocation = "refs/git-p4-tmp"
2592         self.largeFileSystem = None
2593         self.suppress_meta_comment = False
2594
2595         if gitConfig('git-p4.largeFileSystem'):
2596             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2597             self.largeFileSystem = largeFileSystemConstructor(
2598                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2599             )
2600
2601         if gitConfig("git-p4.syncFromOrigin") == "false":
2602             self.syncWithOrigin = False
2603
2604         self.depotPaths = []
2605         self.changeRange = ""
2606         self.previousDepotPaths = []
2607         self.hasOrigin = False
2608
2609         # map from branch depot path to parent branch
2610         self.knownBranches = {}
2611         self.initialParents = {}
2612
2613         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2614         self.labels = {}
2615
2616     # Force a checkpoint in fast-import and wait for it to finish
2617     def checkpoint(self):
2618         self.gitStream.write("checkpoint\n\n")
2619         self.gitStream.write("progress checkpoint\n\n")
2620         out = self.gitOutput.readline()
2621         if self.verbose:
2622             print("checkpoint finished: " + out)
2623
2624     def isPathWanted(self, path):
2625         for p in self.cloneExclude:
2626             if p.endswith("/"):
2627                 if p4PathStartsWith(path, p):
2628                     return False
2629             # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2630             elif path.lower() == p.lower():
2631                 return False
2632         for p in self.depotPaths:
2633             if p4PathStartsWith(path, p):
2634                 return True
2635         return False
2636
2637     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2638         files = []
2639         fnum = 0
2640         while "depotFile%s" % fnum in commit:
2641             path =  commit["depotFile%s" % fnum]
2642             found = self.isPathWanted(path)
2643             if not found:
2644                 fnum = fnum + 1
2645                 continue
2646
2647             file = {}
2648             file["path"] = path
2649             file["rev"] = commit["rev%s" % fnum]
2650             file["action"] = commit["action%s" % fnum]
2651             file["type"] = commit["type%s" % fnum]
2652             if shelved:
2653                 file["shelved_cl"] = int(shelved_cl)
2654             files.append(file)
2655             fnum = fnum + 1
2656         return files
2657
2658     def extractJobsFromCommit(self, commit):
2659         jobs = []
2660         jnum = 0
2661         while "job%s" % jnum in commit:
2662             job = commit["job%s" % jnum]
2663             jobs.append(job)
2664             jnum = jnum + 1
2665         return jobs
2666
2667     def stripRepoPath(self, path, prefixes):
2668         """When streaming files, this is called to map a p4 depot path
2669            to where it should go in git.  The prefixes are either
2670            self.depotPaths, or self.branchPrefixes in the case of
2671            branch detection."""
2672
2673         if self.useClientSpec:
2674             # branch detection moves files up a level (the branch name)
2675             # from what client spec interpretation gives
2676             path = self.clientSpecDirs.map_in_client(path)
2677             if self.detectBranches:
2678                 for b in self.knownBranches:
2679                     if p4PathStartsWith(path, b + "/"):
2680                         path = path[len(b)+1:]
2681
2682         elif self.keepRepoPath:
2683             # Preserve everything in relative path name except leading
2684             # //depot/; just look at first prefix as they all should
2685             # be in the same depot.
2686             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2687             if p4PathStartsWith(path, depot):
2688                 path = path[len(depot):]
2689
2690         else:
2691             for p in prefixes:
2692                 if p4PathStartsWith(path, p):
2693                     path = path[len(p):]
2694                     break
2695
2696         path = wildcard_decode(path)
2697         return path
2698
2699     def splitFilesIntoBranches(self, commit):
2700         """Look at each depotFile in the commit to figure out to what
2701            branch it belongs."""
2702
2703         if self.clientSpecDirs:
2704             files = self.extractFilesFromCommit(commit)
2705             self.clientSpecDirs.update_client_spec_path_cache(files)
2706
2707         branches = {}
2708         fnum = 0
2709         while "depotFile%s" % fnum in commit:
2710             path =  commit["depotFile%s" % fnum]
2711             found = self.isPathWanted(path)
2712             if not found:
2713                 fnum = fnum + 1
2714                 continue
2715
2716             file = {}
2717             file["path"] = path
2718             file["rev"] = commit["rev%s" % fnum]
2719             file["action"] = commit["action%s" % fnum]
2720             file["type"] = commit["type%s" % fnum]
2721             fnum = fnum + 1
2722
2723             # start with the full relative path where this file would
2724             # go in a p4 client
2725             if self.useClientSpec:
2726                 relPath = self.clientSpecDirs.map_in_client(path)
2727             else:
2728                 relPath = self.stripRepoPath(path, self.depotPaths)
2729
2730             for branch in self.knownBranches.keys():
2731                 # add a trailing slash so that a commit into qt/4.2foo
2732                 # doesn't end up in qt/4.2, e.g.
2733                 if p4PathStartsWith(relPath, branch + "/"):
2734                     if branch not in branches:
2735                         branches[branch] = []
2736                     branches[branch].append(file)
2737                     break
2738
2739         return branches
2740
2741     def writeToGitStream(self, gitMode, relPath, contents):
2742         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2743         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2744         for d in contents:
2745             self.gitStream.write(d)
2746         self.gitStream.write('\n')
2747
2748     def encodeWithUTF8(self, path):
2749         try:
2750             path.decode('ascii')
2751         except:
2752             encoding = 'utf8'
2753             if gitConfig('git-p4.pathEncoding'):
2754                 encoding = gitConfig('git-p4.pathEncoding')
2755             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2756             if self.verbose:
2757                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2758         return path
2759
2760     # output one file from the P4 stream
2761     # - helper for streamP4Files
2762
2763     def streamOneP4File(self, file, contents):
2764         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2765         relPath = self.encodeWithUTF8(relPath)
2766         if verbose:
2767             if 'fileSize' in self.stream_file:
2768                 size = int(self.stream_file['fileSize'])
2769             else:
2770                 size = 0 # deleted files don't get a fileSize apparently
2771             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2772             sys.stdout.flush()
2773
2774         (type_base, type_mods) = split_p4_type(file["type"])
2775
2776         git_mode = "100644"
2777         if "x" in type_mods:
2778             git_mode = "100755"
2779         if type_base == "symlink":
2780             git_mode = "120000"
2781             # p4 print on a symlink sometimes contains "target\n";
2782             # if it does, remove the newline
2783             data = ''.join(contents)
2784             if not data:
2785                 # Some version of p4 allowed creating a symlink that pointed
2786                 # to nothing.  This causes p4 errors when checking out such
2787                 # a change, and errors here too.  Work around it by ignoring
2788                 # the bad symlink; hopefully a future change fixes it.
2789                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2790                 return
2791             elif data[-1] == '\n':
2792                 contents = [data[:-1]]
2793             else:
2794                 contents = [data]
2795
2796         if type_base == "utf16":
2797             # p4 delivers different text in the python output to -G
2798             # than it does when using "print -o", or normal p4 client
2799             # operations.  utf16 is converted to ascii or utf8, perhaps.
2800             # But ascii text saved as -t utf16 is completely mangled.
2801             # Invoke print -o to get the real contents.
2802             #
2803             # On windows, the newlines will always be mangled by print, so put
2804             # them back too.  This is not needed to the cygwin windows version,
2805             # just the native "NT" type.
2806             #
2807             try:
2808                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2809             except Exception as e:
2810                 if 'Translation of file content failed' in str(e):
2811                     type_base = 'binary'
2812                 else:
2813                     raise e
2814             else:
2815                 if p4_version_string().find('/NT') >= 0:
2816                     text = text.replace('\r\n', '\n')
2817                 contents = [ text ]
2818
2819         if type_base == "apple":
2820             # Apple filetype files will be streamed as a concatenation of
2821             # its appledouble header and the contents.  This is useless
2822             # on both macs and non-macs.  If using "print -q -o xx", it
2823             # will create "xx" with the data, and "%xx" with the header.
2824             # This is also not very useful.
2825             #
2826             # Ideally, someday, this script can learn how to generate
2827             # appledouble files directly and import those to git, but
2828             # non-mac machines can never find a use for apple filetype.
2829             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2830             return
2831
2832         # Note that we do not try to de-mangle keywords on utf16 files,
2833         # even though in theory somebody may want that.
2834         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2835         if pattern:
2836             regexp = re.compile(pattern, re.VERBOSE)
2837             text = ''.join(contents)
2838             text = regexp.sub(r'$\1$', text)
2839             contents = [ text ]
2840
2841         if self.largeFileSystem:
2842             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2843
2844         self.writeToGitStream(git_mode, relPath, contents)
2845
2846     def streamOneP4Deletion(self, file):
2847         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2848         relPath = self.encodeWithUTF8(relPath)
2849         if verbose:
2850             sys.stdout.write("delete %s\n" % relPath)
2851             sys.stdout.flush()
2852         self.gitStream.write("D %s\n" % relPath)
2853
2854         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2855             self.largeFileSystem.removeLargeFile(relPath)
2856
2857     # handle another chunk of streaming data
2858     def streamP4FilesCb(self, marshalled):
2859
2860         # catch p4 errors and complain
2861         err = None
2862         if "code" in marshalled:
2863             if marshalled["code"] == "error":
2864                 if "data" in marshalled:
2865                     err = marshalled["data"].rstrip()
2866
2867         if not err and 'fileSize' in self.stream_file:
2868             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2869             if required_bytes > 0:
2870                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2871                     os.getcwd(), required_bytes/1024/1024
2872                 )
2873
2874         if err:
2875             f = None
2876             if self.stream_have_file_info:
2877                 if "depotFile" in self.stream_file:
2878                     f = self.stream_file["depotFile"]
2879             # force a failure in fast-import, else an empty
2880             # commit will be made
2881             self.gitStream.write("\n")
2882             self.gitStream.write("die-now\n")
2883             self.gitStream.close()
2884             # ignore errors, but make sure it exits first
2885             self.importProcess.wait()
2886             if f:
2887                 die("Error from p4 print for %s: %s" % (f, err))
2888             else:
2889                 die("Error from p4 print: %s" % err)
2890
2891         if 'depotFile' in marshalled and self.stream_have_file_info:
2892             # start of a new file - output the old one first
2893             self.streamOneP4File(self.stream_file, self.stream_contents)
2894             self.stream_file = {}
2895             self.stream_contents = []
2896             self.stream_have_file_info = False
2897
2898         # pick up the new file information... for the
2899         # 'data' field we need to append to our array
2900         for k in marshalled.keys():
2901             if k == 'data':
2902                 if 'streamContentSize' not in self.stream_file:
2903                     self.stream_file['streamContentSize'] = 0
2904                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2905                 self.stream_contents.append(marshalled['data'])
2906             else:
2907                 self.stream_file[k] = marshalled[k]
2908
2909         if (verbose and
2910             'streamContentSize' in self.stream_file and
2911             'fileSize' in self.stream_file and
2912             'depotFile' in self.stream_file):
2913             size = int(self.stream_file["fileSize"])
2914             if size > 0:
2915                 progress = 100*self.stream_file['streamContentSize']/size
2916                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2917                 sys.stdout.flush()
2918
2919         self.stream_have_file_info = True
2920
2921     # Stream directly from "p4 files" into "git fast-import"
2922     def streamP4Files(self, files):
2923         filesForCommit = []
2924         filesToRead = []
2925         filesToDelete = []
2926
2927         for f in files:
2928             filesForCommit.append(f)
2929             if f['action'] in self.delete_actions:
2930                 filesToDelete.append(f)
2931             else:
2932                 filesToRead.append(f)
2933
2934         # deleted files...
2935         for f in filesToDelete:
2936             self.streamOneP4Deletion(f)
2937
2938         if len(filesToRead) > 0:
2939             self.stream_file = {}
2940             self.stream_contents = []
2941             self.stream_have_file_info = False
2942
2943             # curry self argument
2944             def streamP4FilesCbSelf(entry):
2945                 self.streamP4FilesCb(entry)
2946
2947             fileArgs = []
2948             for f in filesToRead:
2949                 if 'shelved_cl' in f:
2950                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2951                     # the contents
2952                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2953                 else:
2954                     fileArg = '%s#%s' % (f['path'], f['rev'])
2955
2956                 fileArgs.append(fileArg)
2957
2958             p4CmdList(["-x", "-", "print"],
2959                       stdin=fileArgs,
2960                       cb=streamP4FilesCbSelf)
2961
2962             # do the last chunk
2963             if 'depotFile' in self.stream_file:
2964                 self.streamOneP4File(self.stream_file, self.stream_contents)
2965
2966     def make_email(self, userid):
2967         if userid in self.users:
2968             return self.users[userid]
2969         else:
2970             return "%s <a@b>" % userid
2971
2972     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2973         """ Stream a p4 tag.
2974         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2975         """
2976
2977         if verbose:
2978             print("writing tag %s for commit %s" % (labelName, commit))
2979         gitStream.write("tag %s\n" % labelName)
2980         gitStream.write("from %s\n" % commit)
2981
2982         if 'Owner' in labelDetails:
2983             owner = labelDetails["Owner"]
2984         else:
2985             owner = None
2986
2987         # Try to use the owner of the p4 label, or failing that,
2988         # the current p4 user id.
2989         if owner:
2990             email = self.make_email(owner)
2991         else:
2992             email = self.make_email(self.p4UserId())
2993         tagger = "%s %s %s" % (email, epoch, self.tz)
2994
2995         gitStream.write("tagger %s\n" % tagger)
2996
2997         print("labelDetails=",labelDetails)
2998         if 'Description' in labelDetails:
2999             description = labelDetails['Description']
3000         else:
3001             description = 'Label from git p4'
3002
3003         gitStream.write("data %d\n" % len(description))
3004         gitStream.write(description)
3005         gitStream.write("\n")
3006
3007     def inClientSpec(self, path):
3008         if not self.clientSpecDirs:
3009             return True
3010         inClientSpec = self.clientSpecDirs.map_in_client(path)
3011         if not inClientSpec and self.verbose:
3012             print('Ignoring file outside of client spec: {0}'.format(path))
3013         return inClientSpec
3014
3015     def hasBranchPrefix(self, path):
3016         if not self.branchPrefixes:
3017             return True
3018         hasPrefix = [p for p in self.branchPrefixes
3019                         if p4PathStartsWith(path, p)]
3020         if not hasPrefix and self.verbose:
3021             print('Ignoring file outside of prefix: {0}'.format(path))
3022         return hasPrefix
3023
3024     def commit(self, details, files, branch, parent = "", allow_empty=False):
3025         epoch = details["time"]
3026         author = details["user"]
3027         jobs = self.extractJobsFromCommit(details)
3028
3029         if self.verbose:
3030             print('commit into {0}'.format(branch))
3031
3032         if self.clientSpecDirs:
3033             self.clientSpecDirs.update_client_spec_path_cache(files)
3034
3035         files = [f for f in files
3036             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3037
3038         if gitConfigBool('git-p4.keepEmptyCommits'):
3039             allow_empty = True
3040
3041         if not files and not allow_empty:
3042             print('Ignoring revision {0} as it would produce an empty commit.'
3043                 .format(details['change']))
3044             return
3045
3046         self.gitStream.write("commit %s\n" % branch)
3047         self.gitStream.write("mark :%s\n" % details["change"])
3048         self.committedChanges.add(int(details["change"]))
3049         committer = ""
3050         if author not in self.users:
3051             self.getUserMapFromPerforceServer()
3052         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3053
3054         self.gitStream.write("committer %s\n" % committer)
3055
3056         self.gitStream.write("data <<EOT\n")
3057         self.gitStream.write(details["desc"])
3058         if len(jobs) > 0:
3059             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3060
3061         if not self.suppress_meta_comment:
3062             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3063                                 (','.join(self.branchPrefixes), details["change"]))
3064             if len(details['options']) > 0:
3065                 self.gitStream.write(": options = %s" % details['options'])
3066             self.gitStream.write("]\n")
3067
3068         self.gitStream.write("EOT\n\n")
3069
3070         if len(parent) > 0:
3071             if self.verbose:
3072                 print("parent %s" % parent)
3073             self.gitStream.write("from %s\n" % parent)
3074
3075         self.streamP4Files(files)
3076         self.gitStream.write("\n")
3077
3078         change = int(details["change"])
3079
3080         if change in self.labels:
3081             label = self.labels[change]
3082             labelDetails = label[0]
3083             labelRevisions = label[1]
3084             if self.verbose:
3085                 print("Change %s is labelled %s" % (change, labelDetails))
3086
3087             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3088                                                 for p in self.branchPrefixes])
3089
3090             if len(files) == len(labelRevisions):
3091
3092                 cleanedFiles = {}
3093                 for info in files:
3094                     if info["action"] in self.delete_actions:
3095                         continue
3096                     cleanedFiles[info["depotFile"]] = info["rev"]
3097
3098                 if cleanedFiles == labelRevisions:
3099                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3100
3101                 else:
3102                     if not self.silent:
3103                         print("Tag %s does not match with change %s: files do not match."
3104                                % (labelDetails["label"], change))
3105
3106             else:
3107                 if not self.silent:
3108                     print("Tag %s does not match with change %s: file count is different."
3109                            % (labelDetails["label"], change))
3110
3111     # Build a dictionary of changelists and labels, for "detect-labels" option.
3112     def getLabels(self):
3113         self.labels = {}
3114
3115         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3116         if len(l) > 0 and not self.silent:
3117             print("Finding files belonging to labels in %s" % self.depotPaths)
3118
3119         for output in l:
3120             label = output["label"]
3121             revisions = {}
3122             newestChange = 0
3123             if self.verbose:
3124                 print("Querying files for label %s" % label)
3125             for file in p4CmdList(["files"] +
3126                                       ["%s...@%s" % (p, label)
3127                                           for p in self.depotPaths]):
3128                 revisions[file["depotFile"]] = file["rev"]
3129                 change = int(file["change"])
3130                 if change > newestChange:
3131                     newestChange = change
3132
3133             self.labels[newestChange] = [output, revisions]
3134
3135         if self.verbose:
3136             print("Label changes: %s" % self.labels.keys())
3137
3138     # Import p4 labels as git tags. A direct mapping does not
3139     # exist, so assume that if all the files are at the same revision
3140     # then we can use that, or it's something more complicated we should
3141     # just ignore.
3142     def importP4Labels(self, stream, p4Labels):
3143         if verbose:
3144             print("import p4 labels: " + ' '.join(p4Labels))
3145
3146         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3147         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3148         if len(validLabelRegexp) == 0:
3149             validLabelRegexp = defaultLabelRegexp
3150         m = re.compile(validLabelRegexp)
3151
3152         for name in p4Labels:
3153             commitFound = False
3154
3155             if not m.match(name):
3156                 if verbose:
3157                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3158                 continue
3159
3160             if name in ignoredP4Labels:
3161                 continue
3162
3163             labelDetails = p4CmdList(['label', "-o", name])[0]
3164
3165             # get the most recent changelist for each file in this label
3166             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3167                                 for p in self.depotPaths])
3168
3169             if 'change' in change:
3170                 # find the corresponding git commit; take the oldest commit
3171                 changelist = int(change['change'])
3172                 if changelist in self.committedChanges:
3173                     gitCommit = ":%d" % changelist       # use a fast-import mark
3174                     commitFound = True
3175                 else:
3176                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3177                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3178                     if len(gitCommit) == 0:
3179                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3180                     else:
3181                         commitFound = True
3182                         gitCommit = gitCommit.strip()
3183
3184                 if commitFound:
3185                     # Convert from p4 time format
3186                     try:
3187                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3188                     except ValueError:
3189                         print("Could not convert label time %s" % labelDetails['Update'])
3190                         tmwhen = 1
3191
3192                     when = int(time.mktime(tmwhen))
3193                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3194                     if verbose:
3195                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3196             else:
3197                 if verbose:
3198                     print("Label %s has no changelists - possibly deleted?" % name)
3199
3200             if not commitFound:
3201                 # We can't import this label; don't try again as it will get very
3202                 # expensive repeatedly fetching all the files for labels that will
3203                 # never be imported. If the label is moved in the future, the
3204                 # ignore will need to be removed manually.
3205                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3206
3207     def guessProjectName(self):
3208         for p in self.depotPaths:
3209             if p.endswith("/"):
3210                 p = p[:-1]
3211             p = p[p.strip().rfind("/") + 1:]
3212             if not p.endswith("/"):
3213                p += "/"
3214             return p
3215
3216     def getBranchMapping(self):
3217         lostAndFoundBranches = set()
3218
3219         user = gitConfig("git-p4.branchUser")
3220         if len(user) > 0:
3221             command = "branches -u %s" % user
3222         else:
3223             command = "branches"
3224
3225         for info in p4CmdList(command):
3226             details = p4Cmd(["branch", "-o", info["branch"]])
3227             viewIdx = 0
3228             while "View%s" % viewIdx in details:
3229                 paths = details["View%s" % viewIdx].split(" ")
3230                 viewIdx = viewIdx + 1
3231                 # require standard //depot/foo/... //depot/bar/... mapping
3232                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3233                     continue
3234                 source = paths[0]
3235                 destination = paths[1]
3236                 ## HACK
3237                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3238                     source = source[len(self.depotPaths[0]):-4]
3239                     destination = destination[len(self.depotPaths[0]):-4]
3240
3241                     if destination in self.knownBranches:
3242                         if not self.silent:
3243                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3244                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3245                         continue
3246
3247                     self.knownBranches[destination] = source
3248
3249                     lostAndFoundBranches.discard(destination)
3250
3251                     if source not in self.knownBranches:
3252                         lostAndFoundBranches.add(source)
3253
3254         # Perforce does not strictly require branches to be defined, so we also
3255         # check git config for a branch list.
3256         #
3257         # Example of branch definition in git config file:
3258         # [git-p4]
3259         #   branchList=main:branchA
3260         #   branchList=main:branchB
3261         #   branchList=branchA:branchC
3262         configBranches = gitConfigList("git-p4.branchList")
3263         for branch in configBranches:
3264             if branch:
3265                 (source, destination) = branch.split(":")
3266                 self.knownBranches[destination] = source
3267
3268                 lostAndFoundBranches.discard(destination)
3269
3270                 if source not in self.knownBranches:
3271                     lostAndFoundBranches.add(source)
3272
3273
3274         for branch in lostAndFoundBranches:
3275             self.knownBranches[branch] = branch
3276
3277     def getBranchMappingFromGitBranches(self):
3278         branches = p4BranchesInGit(self.importIntoRemotes)
3279         for branch in branches.keys():
3280             if branch == "master":
3281                 branch = "main"
3282             else:
3283                 branch = branch[len(self.projectName):]
3284             self.knownBranches[branch] = branch
3285
3286     def updateOptionDict(self, d):
3287         option_keys = {}
3288         if self.keepRepoPath:
3289             option_keys['keepRepoPath'] = 1
3290
3291         d["options"] = ' '.join(sorted(option_keys.keys()))
3292
3293     def readOptions(self, d):
3294         self.keepRepoPath = ('options' in d
3295                              and ('keepRepoPath' in d['options']))
3296
3297     def gitRefForBranch(self, branch):
3298         if branch == "main":
3299             return self.refPrefix + "master"
3300
3301         if len(branch) <= 0:
3302             return branch
3303
3304         return self.refPrefix + self.projectName + branch
3305
3306     def gitCommitByP4Change(self, ref, change):
3307         if self.verbose:
3308             print("looking in ref " + ref + " for change %s using bisect..." % change)
3309
3310         earliestCommit = ""
3311         latestCommit = parseRevision(ref)
3312
3313         while True:
3314             if self.verbose:
3315                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3316             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3317             if len(next) == 0:
3318                 if self.verbose:
3319                     print("argh")
3320                 return ""
3321             log = extractLogMessageFromGitCommit(next)
3322             settings = extractSettingsGitLog(log)
3323             currentChange = int(settings['change'])
3324             if self.verbose:
3325                 print("current change %s" % currentChange)
3326
3327             if currentChange == change:
3328                 if self.verbose:
3329                     print("found %s" % next)
3330                 return next
3331
3332             if currentChange < change:
3333                 earliestCommit = "^%s" % next
3334             else:
3335                 if next == latestCommit:
3336                     die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3337                 latestCommit = "%s^@" % next
3338
3339         return ""
3340
3341     def importNewBranch(self, branch, maxChange):
3342         # make fast-import flush all changes to disk and update the refs using the checkpoint
3343         # command so that we can try to find the branch parent in the git history
3344         self.gitStream.write("checkpoint\n\n");
3345         self.gitStream.flush();
3346         branchPrefix = self.depotPaths[0] + branch + "/"
3347         range = "@1,%s" % maxChange
3348         #print "prefix" + branchPrefix
3349         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3350         if len(changes) <= 0:
3351             return False
3352         firstChange = changes[0]
3353         #print "first change in branch: %s" % firstChange
3354         sourceBranch = self.knownBranches[branch]
3355         sourceDepotPath = self.depotPaths[0] + sourceBranch
3356         sourceRef = self.gitRefForBranch(sourceBranch)
3357         #print "source " + sourceBranch
3358
3359         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3360         #print "branch parent: %s" % branchParentChange
3361         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3362         if len(gitParent) > 0:
3363             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3364             #print "parent git commit: %s" % gitParent
3365
3366         self.importChanges(changes)
3367         return True
3368
3369     def searchParent(self, parent, branch, target):
3370         parentFound = False
3371         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3372                                      "--no-merges", parent]):
3373             blob = blob.strip()
3374             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3375                 parentFound = True
3376                 if self.verbose:
3377                     print("Found parent of %s in commit %s" % (branch, blob))
3378                 break
3379         if parentFound:
3380             return blob
3381         else:
3382             return None
3383
3384     def importChanges(self, changes, origin_revision=0):
3385         cnt = 1
3386         for change in changes:
3387             description = p4_describe(change)
3388             self.updateOptionDict(description)
3389
3390             if not self.silent:
3391                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3392                 sys.stdout.flush()
3393             cnt = cnt + 1
3394
3395             try:
3396                 if self.detectBranches:
3397                     branches = self.splitFilesIntoBranches(description)
3398                     for branch in branches.keys():
3399                         ## HACK  --hwn
3400                         branchPrefix = self.depotPaths[0] + branch + "/"
3401                         self.branchPrefixes = [ branchPrefix ]
3402
3403                         parent = ""
3404
3405                         filesForCommit = branches[branch]
3406
3407                         if self.verbose:
3408                             print("branch is %s" % branch)
3409
3410                         self.updatedBranches.add(branch)
3411
3412                         if branch not in self.createdBranches:
3413                             self.createdBranches.add(branch)
3414                             parent = self.knownBranches[branch]
3415                             if parent == branch:
3416                                 parent = ""
3417                             else:
3418                                 fullBranch = self.projectName + branch
3419                                 if fullBranch not in self.p4BranchesInGit:
3420                                     if not self.silent:
3421                                         print("\n    Importing new branch %s" % fullBranch);
3422                                     if self.importNewBranch(branch, change - 1):
3423                                         parent = ""
3424                                         self.p4BranchesInGit.append(fullBranch)
3425                                     if not self.silent:
3426                                         print("\n    Resuming with change %s" % change);
3427
3428                                 if self.verbose:
3429                                     print("parent determined through known branches: %s" % parent)
3430
3431                         branch = self.gitRefForBranch(branch)
3432                         parent = self.gitRefForBranch(parent)
3433
3434                         if self.verbose:
3435                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3436
3437                         if len(parent) == 0 and branch in self.initialParents:
3438                             parent = self.initialParents[branch]
3439                             del self.initialParents[branch]
3440
3441                         blob = None
3442                         if len(parent) > 0:
3443                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3444                             if self.verbose:
3445                                 print("Creating temporary branch: " + tempBranch)
3446                             self.commit(description, filesForCommit, tempBranch)
3447                             self.tempBranches.append(tempBranch)
3448                             self.checkpoint()
3449                             blob = self.searchParent(parent, branch, tempBranch)
3450                         if blob:
3451                             self.commit(description, filesForCommit, branch, blob)
3452                         else:
3453                             if self.verbose:
3454                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3455                             self.commit(description, filesForCommit, branch, parent)
3456                 else:
3457                     files = self.extractFilesFromCommit(description)
3458                     self.commit(description, files, self.branch,
3459                                 self.initialParent)
3460                     # only needed once, to connect to the previous commit
3461                     self.initialParent = ""
3462             except IOError:
3463                 print(self.gitError.read())
3464                 sys.exit(1)
3465
3466     def sync_origin_only(self):
3467         if self.syncWithOrigin:
3468             self.hasOrigin = originP4BranchesExist()
3469             if self.hasOrigin:
3470                 if not self.silent:
3471                     print('Syncing with origin first, using "git fetch origin"')
3472                 system("git fetch origin")
3473
3474     def importHeadRevision(self, revision):
3475         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3476
3477         details = {}
3478         details["user"] = "git perforce import user"
3479         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3480                            % (' '.join(self.depotPaths), revision))
3481         details["change"] = revision
3482         newestRevision = 0
3483
3484         fileCnt = 0
3485         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3486
3487         for info in p4CmdList(["files"] + fileArgs):
3488
3489             if 'code' in info and info['code'] == 'error':
3490                 sys.stderr.write("p4 returned an error: %s\n"
3491                                  % info['data'])
3492                 if info['data'].find("must refer to client") >= 0:
3493                     sys.stderr.write("This particular p4 error is misleading.\n")
3494                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3495                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3496                 sys.exit(1)
3497             if 'p4ExitCode' in info:
3498                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3499                 sys.exit(1)
3500
3501
3502             change = int(info["change"])
3503             if change > newestRevision:
3504                 newestRevision = change
3505
3506             if info["action"] in self.delete_actions:
3507                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3508                 #fileCnt = fileCnt + 1
3509                 continue
3510
3511             for prop in ["depotFile", "rev", "action", "type" ]:
3512                 details["%s%s" % (prop, fileCnt)] = info[prop]
3513
3514             fileCnt = fileCnt + 1
3515
3516         details["change"] = newestRevision
3517
3518         # Use time from top-most change so that all git p4 clones of
3519         # the same p4 repo have the same commit SHA1s.
3520         res = p4_describe(newestRevision)
3521         details["time"] = res["time"]
3522
3523         self.updateOptionDict(details)
3524         try:
3525             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3526         except IOError as err:
3527             print("IO error with git fast-import. Is your git version recent enough?")
3528             print("IO error details: {}".format(err))
3529             print(self.gitError.read())
3530
3531     def openStreams(self):
3532         self.importProcess = subprocess.Popen(["git", "fast-import"],
3533                                               stdin=subprocess.PIPE,
3534                                               stdout=subprocess.PIPE,
3535                                               stderr=subprocess.PIPE);
3536         self.gitOutput = self.importProcess.stdout
3537         self.gitStream = self.importProcess.stdin
3538         self.gitError = self.importProcess.stderr
3539
3540     def closeStreams(self):
3541         self.gitStream.close()
3542         if self.importProcess.wait() != 0:
3543             die("fast-import failed: %s" % self.gitError.read())
3544         self.gitOutput.close()
3545         self.gitError.close()
3546
3547     def run(self, args):
3548         if self.importIntoRemotes:
3549             self.refPrefix = "refs/remotes/p4/"
3550         else:
3551             self.refPrefix = "refs/heads/p4/"
3552
3553         self.sync_origin_only()
3554
3555         branch_arg_given = bool(self.branch)
3556         if len(self.branch) == 0:
3557             self.branch = self.refPrefix + "master"
3558             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3559                 system("git update-ref %s refs/heads/p4" % self.branch)
3560                 system("git branch -D p4")
3561
3562         # accept either the command-line option, or the configuration variable
3563         if self.useClientSpec:
3564             # will use this after clone to set the variable
3565             self.useClientSpec_from_options = True
3566         else:
3567             if gitConfigBool("git-p4.useclientspec"):
3568                 self.useClientSpec = True
3569         if self.useClientSpec:
3570             self.clientSpecDirs = getClientSpec()
3571
3572         # TODO: should always look at previous commits,
3573         # merge with previous imports, if possible.
3574         if args == []:
3575             if self.hasOrigin:
3576                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3577
3578             # branches holds mapping from branch name to sha1
3579             branches = p4BranchesInGit(self.importIntoRemotes)
3580
3581             # restrict to just this one, disabling detect-branches
3582             if branch_arg_given:
3583                 short = self.branch.split("/")[-1]
3584                 if short in branches:
3585                     self.p4BranchesInGit = [ short ]
3586             else:
3587                 self.p4BranchesInGit = branches.keys()
3588
3589             if len(self.p4BranchesInGit) > 1:
3590                 if not self.silent:
3591                     print("Importing from/into multiple branches")
3592                 self.detectBranches = True
3593                 for branch in branches.keys():
3594                     self.initialParents[self.refPrefix + branch] = \
3595                         branches[branch]
3596
3597             if self.verbose:
3598                 print("branches: %s" % self.p4BranchesInGit)
3599
3600             p4Change = 0
3601             for branch in self.p4BranchesInGit:
3602                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3603
3604                 settings = extractSettingsGitLog(logMsg)
3605
3606                 self.readOptions(settings)
3607                 if ('depot-paths' in settings
3608                     and 'change' in settings):
3609                     change = int(settings['change']) + 1
3610                     p4Change = max(p4Change, change)
3611
3612                     depotPaths = sorted(settings['depot-paths'])
3613                     if self.previousDepotPaths == []:
3614                         self.previousDepotPaths = depotPaths
3615                     else:
3616                         paths = []
3617                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3618                             prev_list = prev.split("/")
3619                             cur_list = cur.split("/")
3620                             for i in range(0, min(len(cur_list), len(prev_list))):
3621                                 if cur_list[i] != prev_list[i]:
3622                                     i = i - 1
3623                                     break
3624
3625                             paths.append ("/".join(cur_list[:i + 1]))
3626
3627                         self.previousDepotPaths = paths
3628
3629             if p4Change > 0:
3630                 self.depotPaths = sorted(self.previousDepotPaths)
3631                 self.changeRange = "@%s,#head" % p4Change
3632                 if not self.silent and not self.detectBranches:
3633                     print("Performing incremental import into %s git branch" % self.branch)
3634
3635         # accept multiple ref name abbreviations:
3636         #    refs/foo/bar/branch -> use it exactly
3637         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3638         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3639         if not self.branch.startswith("refs/"):
3640             if self.importIntoRemotes:
3641                 prepend = "refs/remotes/"
3642             else:
3643                 prepend = "refs/heads/"
3644             if not self.branch.startswith("p4/"):
3645                 prepend += "p4/"
3646             self.branch = prepend + self.branch
3647
3648         if len(args) == 0 and self.depotPaths:
3649             if not self.silent:
3650                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3651         else:
3652             if self.depotPaths and self.depotPaths != args:
3653                 print("previous import used depot path %s and now %s was specified. "
3654                        "This doesn't work!" % (' '.join (self.depotPaths),
3655                                                ' '.join (args)))
3656                 sys.exit(1)
3657
3658             self.depotPaths = sorted(args)
3659
3660         revision = ""
3661         self.users = {}
3662
3663         # Make sure no revision specifiers are used when --changesfile
3664         # is specified.
3665         bad_changesfile = False
3666         if len(self.changesFile) > 0:
3667             for p in self.depotPaths:
3668                 if p.find("@") >= 0 or p.find("#") >= 0:
3669                     bad_changesfile = True
3670                     break
3671         if bad_changesfile:
3672             die("Option --changesfile is incompatible with revision specifiers")
3673
3674         newPaths = []
3675         for p in self.depotPaths:
3676             if p.find("@") != -1:
3677                 atIdx = p.index("@")
3678                 self.changeRange = p[atIdx:]
3679                 if self.changeRange == "@all":
3680                     self.changeRange = ""
3681                 elif ',' not in self.changeRange:
3682                     revision = self.changeRange
3683                     self.changeRange = ""
3684                 p = p[:atIdx]
3685             elif p.find("#") != -1:
3686                 hashIdx = p.index("#")
3687                 revision = p[hashIdx:]
3688                 p = p[:hashIdx]
3689             elif self.previousDepotPaths == []:
3690                 # pay attention to changesfile, if given, else import
3691                 # the entire p4 tree at the head revision
3692                 if len(self.changesFile) == 0:
3693                     revision = "#head"
3694
3695             p = re.sub ("\.\.\.$", "", p)
3696             if not p.endswith("/"):
3697                 p += "/"
3698
3699             newPaths.append(p)
3700
3701         self.depotPaths = newPaths
3702
3703         # --detect-branches may change this for each branch
3704         self.branchPrefixes = self.depotPaths
3705
3706         self.loadUserMapFromCache()
3707         self.labels = {}
3708         if self.detectLabels:
3709             self.getLabels();
3710
3711         if self.detectBranches:
3712             ## FIXME - what's a P4 projectName ?
3713             self.projectName = self.guessProjectName()
3714
3715             if self.hasOrigin:
3716                 self.getBranchMappingFromGitBranches()
3717             else:
3718                 self.getBranchMapping()
3719             if self.verbose:
3720                 print("p4-git branches: %s" % self.p4BranchesInGit)
3721                 print("initial parents: %s" % self.initialParents)
3722             for b in self.p4BranchesInGit:
3723                 if b != "master":
3724
3725                     ## FIXME
3726                     b = b[len(self.projectName):]
3727                 self.createdBranches.add(b)
3728
3729         self.openStreams()
3730
3731         if revision:
3732             self.importHeadRevision(revision)
3733         else:
3734             changes = []
3735
3736             if len(self.changesFile) > 0:
3737                 output = open(self.changesFile).readlines()
3738                 changeSet = set()
3739                 for line in output:
3740                     changeSet.add(int(line))
3741
3742                 for change in changeSet:
3743                     changes.append(change)
3744
3745                 changes.sort()
3746             else:
3747                 # catch "git p4 sync" with no new branches, in a repo that
3748                 # does not have any existing p4 branches
3749                 if len(args) == 0:
3750                     if not self.p4BranchesInGit:
3751                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3752
3753                     # The default branch is master, unless --branch is used to
3754                     # specify something else.  Make sure it exists, or complain
3755                     # nicely about how to use --branch.
3756                     if not self.detectBranches:
3757                         if not branch_exists(self.branch):
3758                             if branch_arg_given:
3759                                 die("Error: branch %s does not exist." % self.branch)
3760                             else:
3761                                 die("Error: no branch %s; perhaps specify one with --branch." %
3762                                     self.branch)
3763
3764                 if self.verbose:
3765                     print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3766                                                               self.changeRange))
3767                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3768
3769                 if len(self.maxChanges) > 0:
3770                     changes = changes[:min(int(self.maxChanges), len(changes))]
3771
3772             if len(changes) == 0:
3773                 if not self.silent:
3774                     print("No changes to import!")
3775             else:
3776                 if not self.silent and not self.detectBranches:
3777                     print("Import destination: %s" % self.branch)
3778
3779                 self.updatedBranches = set()
3780
3781                 if not self.detectBranches:
3782                     if args:
3783                         # start a new branch
3784                         self.initialParent = ""
3785                     else:
3786                         # build on a previous revision
3787                         self.initialParent = parseRevision(self.branch)
3788
3789                 self.importChanges(changes)
3790
3791                 if not self.silent:
3792                     print("")
3793                     if len(self.updatedBranches) > 0:
3794                         sys.stdout.write("Updated branches: ")
3795                         for b in self.updatedBranches:
3796                             sys.stdout.write("%s " % b)
3797                         sys.stdout.write("\n")
3798
3799         if gitConfigBool("git-p4.importLabels"):
3800             self.importLabels = True
3801
3802         if self.importLabels:
3803             p4Labels = getP4Labels(self.depotPaths)
3804             gitTags = getGitTags()
3805
3806             missingP4Labels = p4Labels - gitTags
3807             self.importP4Labels(self.gitStream, missingP4Labels)
3808
3809         self.closeStreams()
3810
3811         # Cleanup temporary branches created during import
3812         if self.tempBranches != []:
3813             for branch in self.tempBranches:
3814                 read_pipe("git update-ref -d %s" % branch)
3815             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3816
3817         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3818         # a convenient shortcut refname "p4".
3819         if self.importIntoRemotes:
3820             head_ref = self.refPrefix + "HEAD"
3821             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3822                 system(["git", "symbolic-ref", head_ref, self.branch])
3823
3824         return True
3825
3826 class P4Rebase(Command):
3827     def __init__(self):
3828         Command.__init__(self)
3829         self.options = [
3830                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3831         ]
3832         self.importLabels = False
3833         self.description = ("Fetches the latest revision from perforce and "
3834                             + "rebases the current work (branch) against it")
3835
3836     def run(self, args):
3837         sync = P4Sync()
3838         sync.importLabels = self.importLabels
3839         sync.run([])
3840
3841         return self.rebase()
3842
3843     def rebase(self):
3844         if os.system("git update-index --refresh") != 0:
3845             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3846         if len(read_pipe("git diff-index HEAD --")) > 0:
3847             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3848
3849         [upstream, settings] = findUpstreamBranchPoint()
3850         if len(upstream) == 0:
3851             die("Cannot find upstream branchpoint for rebase")
3852
3853         # the branchpoint may be p4/foo~3, so strip off the parent
3854         upstream = re.sub("~[0-9]+$", "", upstream)
3855
3856         print("Rebasing the current branch onto %s" % upstream)
3857         oldHead = read_pipe("git rev-parse HEAD").strip()
3858         system("git rebase %s" % upstream)
3859         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3860         return True
3861
3862 class P4Clone(P4Sync):
3863     def __init__(self):
3864         P4Sync.__init__(self)
3865         self.description = "Creates a new git repository and imports from Perforce into it"
3866         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3867         self.options += [
3868             optparse.make_option("--destination", dest="cloneDestination",
3869                                  action='store', default=None,
3870                                  help="where to leave result of the clone"),
3871             optparse.make_option("--bare", dest="cloneBare",
3872                                  action="store_true", default=False),
3873         ]
3874         self.cloneDestination = None
3875         self.needsGit = False
3876         self.cloneBare = False
3877
3878     def defaultDestination(self, args):
3879         ## TODO: use common prefix of args?
3880         depotPath = args[0]
3881         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3882         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3883         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3884         depotDir = re.sub(r"/$", "", depotDir)
3885         return os.path.split(depotDir)[1]
3886
3887     def run(self, args):
3888         if len(args) < 1:
3889             return False
3890
3891         if self.keepRepoPath and not self.cloneDestination:
3892             sys.stderr.write("Must specify destination for --keep-path\n")
3893             sys.exit(1)
3894
3895         depotPaths = args
3896
3897         if not self.cloneDestination and len(depotPaths) > 1:
3898             self.cloneDestination = depotPaths[-1]
3899             depotPaths = depotPaths[:-1]
3900
3901         for p in depotPaths:
3902             if not p.startswith("//"):
3903                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3904                 return False
3905
3906         if not self.cloneDestination:
3907             self.cloneDestination = self.defaultDestination(args)
3908
3909         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3910
3911         if not os.path.exists(self.cloneDestination):
3912             os.makedirs(self.cloneDestination)
3913         chdir(self.cloneDestination)
3914
3915         init_cmd = [ "git", "init" ]
3916         if self.cloneBare:
3917             init_cmd.append("--bare")
3918         retcode = subprocess.call(init_cmd)
3919         if retcode:
3920             raise CalledProcessError(retcode, init_cmd)
3921
3922         if not P4Sync.run(self, depotPaths):
3923             return False
3924
3925         # create a master branch and check out a work tree
3926         if gitBranchExists(self.branch):
3927             system([ "git", "branch", "master", self.branch ])
3928             if not self.cloneBare:
3929                 system([ "git", "checkout", "-f" ])
3930         else:
3931             print('Not checking out any branch, use ' \
3932                   '"git checkout -q -b master <branch>"')
3933
3934         # auto-set this variable if invoked with --use-client-spec
3935         if self.useClientSpec_from_options:
3936             system("git config --bool git-p4.useclientspec true")
3937
3938         return True
3939
3940 class P4Unshelve(Command):
3941     def __init__(self):
3942         Command.__init__(self)
3943         self.options = []
3944         self.origin = "HEAD"
3945         self.description = "Unshelve a P4 changelist into a git commit"
3946         self.usage = "usage: %prog [options] changelist"
3947         self.options += [
3948                 optparse.make_option("--origin", dest="origin",
3949                     help="Use this base revision instead of the default (%s)" % self.origin),
3950         ]
3951         self.verbose = False
3952         self.noCommit = False
3953         self.destbranch = "refs/remotes/p4-unshelved"
3954
3955     def renameBranch(self, branch_name):
3956         """ Rename the existing branch to branch_name.N
3957         """
3958
3959         found = True
3960         for i in range(0,1000):
3961             backup_branch_name = "{0}.{1}".format(branch_name, i)
3962             if not gitBranchExists(backup_branch_name):
3963                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3964                 gitDeleteRef(branch_name)
3965                 found = True
3966                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3967                 break
3968
3969         if not found:
3970             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3971
3972     def findLastP4Revision(self, starting_point):
3973         """ Look back from starting_point for the first commit created by git-p4
3974             to find the P4 commit we are based on, and the depot-paths.
3975         """
3976
3977         for parent in (range(65535)):
3978             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3979             settings = extractSettingsGitLog(log)
3980             if 'change' in settings:
3981                 return settings
3982
3983         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3984
3985     def createShelveParent(self, change, branch_name, sync, origin):
3986         """ Create a commit matching the parent of the shelved changelist 'change'
3987         """
3988         parent_description = p4_describe(change, shelved=True)
3989         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
3990         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
3991
3992         parent_files = []
3993         for f in files:
3994             # if it was added in the shelved changelist, it won't exist in the parent
3995             if f['action'] in self.add_actions:
3996                 continue
3997
3998             # if it was deleted in the shelved changelist it must not be deleted
3999             # in the parent - we might even need to create it if the origin branch
4000             # does not have it
4001             if f['action'] in self.delete_actions:
4002                 f['action'] = 'add'
4003
4004             parent_files.append(f)
4005
4006         sync.commit(parent_description, parent_files, branch_name,
4007                 parent=origin, allow_empty=True)
4008         print("created parent commit for {0} based on {1} in {2}".format(
4009             change, self.origin, branch_name))
4010
4011     def run(self, args):
4012         if len(args) != 1:
4013             return False
4014
4015         if not gitBranchExists(self.origin):
4016             sys.exit("origin branch {0} does not exist".format(self.origin))
4017
4018         sync = P4Sync()
4019         changes = args
4020
4021         # only one change at a time
4022         change = changes[0]
4023
4024         # if the target branch already exists, rename it
4025         branch_name = "{0}/{1}".format(self.destbranch, change)
4026         if gitBranchExists(branch_name):
4027             self.renameBranch(branch_name)
4028         sync.branch = branch_name
4029
4030         sync.verbose = self.verbose
4031         sync.suppress_meta_comment = True
4032
4033         settings = self.findLastP4Revision(self.origin)
4034         sync.depotPaths = settings['depot-paths']
4035         sync.branchPrefixes = sync.depotPaths
4036
4037         sync.openStreams()
4038         sync.loadUserMapFromCache()
4039         sync.silent = True
4040
4041         # create a commit for the parent of the shelved changelist
4042         self.createShelveParent(change, branch_name, sync, self.origin)
4043
4044         # create the commit for the shelved changelist itself
4045         description = p4_describe(change, True)
4046         files = sync.extractFilesFromCommit(description, True, change)
4047
4048         sync.commit(description, files, branch_name, "")
4049         sync.closeStreams()
4050
4051         print("unshelved changelist {0} into {1}".format(change, branch_name))
4052
4053         return True
4054
4055 class P4Branches(Command):
4056     def __init__(self):
4057         Command.__init__(self)
4058         self.options = [ ]
4059         self.description = ("Shows the git branches that hold imports and their "
4060                             + "corresponding perforce depot paths")
4061         self.verbose = False
4062
4063     def run(self, args):
4064         if originP4BranchesExist():
4065             createOrUpdateBranchesFromOrigin()
4066
4067         cmdline = "git rev-parse --symbolic "
4068         cmdline += " --remotes"
4069
4070         for line in read_pipe_lines(cmdline):
4071             line = line.strip()
4072
4073             if not line.startswith('p4/') or line == "p4/HEAD":
4074                 continue
4075             branch = line
4076
4077             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4078             settings = extractSettingsGitLog(log)
4079
4080             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4081         return True
4082
4083 class HelpFormatter(optparse.IndentedHelpFormatter):
4084     def __init__(self):
4085         optparse.IndentedHelpFormatter.__init__(self)
4086
4087     def format_description(self, description):
4088         if description:
4089             return description + "\n"
4090         else:
4091             return ""
4092
4093 def printUsage(commands):
4094     print("usage: %s <command> [options]" % sys.argv[0])
4095     print("")
4096     print("valid commands: %s" % ", ".join(commands))
4097     print("")
4098     print("Try %s <command> --help for command specific help." % sys.argv[0])
4099     print("")
4100
4101 commands = {
4102     "debug" : P4Debug,
4103     "submit" : P4Submit,
4104     "commit" : P4Submit,
4105     "sync" : P4Sync,
4106     "rebase" : P4Rebase,
4107     "clone" : P4Clone,
4108     "rollback" : P4RollBack,
4109     "branches" : P4Branches,
4110     "unshelve" : P4Unshelve,
4111 }
4112
4113
4114 def main():
4115     if len(sys.argv[1:]) == 0:
4116         printUsage(commands.keys())
4117         sys.exit(2)
4118
4119     cmdName = sys.argv[1]
4120     try:
4121         klass = commands[cmdName]
4122         cmd = klass()
4123     except KeyError:
4124         print("unknown command %s" % cmdName)
4125         print("")
4126         printUsage(commands.keys())
4127         sys.exit(2)
4128
4129     options = cmd.options
4130     cmd.gitdir = os.environ.get("GIT_DIR", None)
4131
4132     args = sys.argv[2:]
4133
4134     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4135     if cmd.needsGit:
4136         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4137
4138     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4139                                    options,
4140                                    description = cmd.description,
4141                                    formatter = HelpFormatter())
4142
4143     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4144     global verbose
4145     verbose = cmd.verbose
4146     if cmd.needsGit:
4147         if cmd.gitdir == None:
4148             cmd.gitdir = os.path.abspath(".git")
4149             if not isValidGitDir(cmd.gitdir):
4150                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4151                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4152                 if os.path.exists(cmd.gitdir):
4153                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4154                     if len(cdup) > 0:
4155                         chdir(cdup);
4156
4157         if not isValidGitDir(cmd.gitdir):
4158             if isValidGitDir(cmd.gitdir + "/.git"):
4159                 cmd.gitdir += "/.git"
4160             else:
4161                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4162
4163         # so git commands invoked from the P4 workspace will succeed
4164         os.environ["GIT_DIR"] = cmd.gitdir
4165
4166     if not cmd.run(args):
4167         parser.print_help()
4168         sys.exit(2)
4169
4170
4171 if __name__ == '__main__':
4172     main()