Merge branch 'jk/make-tags-find-sources-tweak'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28
29 try:
30     from subprocess import CalledProcessError
31 except ImportError:
32     # from python2.7:subprocess.py
33     # Exception classes used by this module.
34     class CalledProcessError(Exception):
35         """This exception is raised when a process run by check_call() returns
36         a non-zero exit status.  The exit status will be stored in the
37         returncode attribute."""
38         def __init__(self, returncode, cmd):
39             self.returncode = returncode
40             self.cmd = cmd
41         def __str__(self):
42             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
44 verbose = False
45
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
48
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
51
52 def p4_build_cmd(cmd):
53     """Build a suitable p4 command line.
54
55     This consolidates building and returning a p4 command line into one
56     location. It means that hooking into the environment, or other configuration
57     can be done more easily.
58     """
59     real_cmd = ["p4"]
60
61     user = gitConfig("git-p4.user")
62     if len(user) > 0:
63         real_cmd += ["-u",user]
64
65     password = gitConfig("git-p4.password")
66     if len(password) > 0:
67         real_cmd += ["-P", password]
68
69     port = gitConfig("git-p4.port")
70     if len(port) > 0:
71         real_cmd += ["-p", port]
72
73     host = gitConfig("git-p4.host")
74     if len(host) > 0:
75         real_cmd += ["-H", host]
76
77     client = gitConfig("git-p4.client")
78     if len(client) > 0:
79         real_cmd += ["-c", client]
80
81     retries = gitConfigInt("git-p4.retries")
82     if retries is None:
83         # Perform 3 retries by default
84         retries = 3
85     real_cmd += ["-r", str(retries)]
86
87     if isinstance(cmd,basestring):
88         real_cmd = ' '.join(real_cmd) + ' ' + cmd
89     else:
90         real_cmd += cmd
91     return real_cmd
92
93 def chdir(path, is_client_path=False):
94     """Do chdir to the given path, and set the PWD environment
95        variable for use by P4.  It does not look at getcwd() output.
96        Since we're not using the shell, it is necessary to set the
97        PWD environment variable explicitly.
98
99        Normally, expand the path to force it to be absolute.  This
100        addresses the use of relative path names inside P4 settings,
101        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
102        as given; it looks for .p4config using PWD.
103
104        If is_client_path, the path was handed to us directly by p4,
105        and may be a symbolic link.  Do not call os.getcwd() in this
106        case, because it will cause p4 to think that PWD is not inside
107        the client path.
108        """
109
110     os.chdir(path)
111     if not is_client_path:
112         path = os.getcwd()
113     os.environ['PWD'] = path
114
115 def calcDiskFree():
116     """Return free space in bytes on the disk of the given dirname."""
117     if platform.system() == 'Windows':
118         free_bytes = ctypes.c_ulonglong(0)
119         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
120         return free_bytes.value
121     else:
122         st = os.statvfs(os.getcwd())
123         return st.f_bavail * st.f_frsize
124
125 def die(msg):
126     if verbose:
127         raise Exception(msg)
128     else:
129         sys.stderr.write(msg + "\n")
130         sys.exit(1)
131
132 def write_pipe(c, stdin):
133     if verbose:
134         sys.stderr.write('Writing pipe: %s\n' % str(c))
135
136     expand = isinstance(c,basestring)
137     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
138     pipe = p.stdin
139     val = pipe.write(stdin)
140     pipe.close()
141     if p.wait():
142         die('Command failed: %s' % str(c))
143
144     return val
145
146 def p4_write_pipe(c, stdin):
147     real_cmd = p4_build_cmd(c)
148     return write_pipe(real_cmd, stdin)
149
150 def read_pipe(c, ignore_error=False):
151     if verbose:
152         sys.stderr.write('Reading pipe: %s\n' % str(c))
153
154     expand = isinstance(c,basestring)
155     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
156     (out, err) = p.communicate()
157     if p.returncode != 0 and not ignore_error:
158         die('Command failed: %s\nError: %s' % (str(c), err))
159     return out
160
161 def p4_read_pipe(c, ignore_error=False):
162     real_cmd = p4_build_cmd(c)
163     return read_pipe(real_cmd, ignore_error)
164
165 def read_pipe_lines(c):
166     if verbose:
167         sys.stderr.write('Reading pipe: %s\n' % str(c))
168
169     expand = isinstance(c, basestring)
170     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
171     pipe = p.stdout
172     val = pipe.readlines()
173     if pipe.close() or p.wait():
174         die('Command failed: %s' % str(c))
175
176     return val
177
178 def p4_read_pipe_lines(c):
179     """Specifically invoke p4 on the command supplied. """
180     real_cmd = p4_build_cmd(c)
181     return read_pipe_lines(real_cmd)
182
183 def p4_has_command(cmd):
184     """Ask p4 for help on this command.  If it returns an error, the
185        command does not exist in this version of p4."""
186     real_cmd = p4_build_cmd(["help", cmd])
187     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
188                                    stderr=subprocess.PIPE)
189     p.communicate()
190     return p.returncode == 0
191
192 def p4_has_move_command():
193     """See if the move command exists, that it supports -k, and that
194        it has not been administratively disabled.  The arguments
195        must be correct, but the filenames do not have to exist.  Use
196        ones with wildcards so even if they exist, it will fail."""
197
198     if not p4_has_command("move"):
199         return False
200     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
201     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
202     (out, err) = p.communicate()
203     # return code will be 1 in either case
204     if err.find("Invalid option") >= 0:
205         return False
206     if err.find("disabled") >= 0:
207         return False
208     # assume it failed because @... was invalid changelist
209     return True
210
211 def system(cmd, ignore_error=False):
212     expand = isinstance(cmd,basestring)
213     if verbose:
214         sys.stderr.write("executing %s\n" % str(cmd))
215     retcode = subprocess.call(cmd, shell=expand)
216     if retcode and not ignore_error:
217         raise CalledProcessError(retcode, cmd)
218
219     return retcode
220
221 def p4_system(cmd):
222     """Specifically invoke p4 as the system command. """
223     real_cmd = p4_build_cmd(cmd)
224     expand = isinstance(real_cmd, basestring)
225     retcode = subprocess.call(real_cmd, shell=expand)
226     if retcode:
227         raise CalledProcessError(retcode, real_cmd)
228
229 _p4_version_string = None
230 def p4_version_string():
231     """Read the version string, showing just the last line, which
232        hopefully is the interesting version bit.
233
234        $ p4 -V
235        Perforce - The Fast Software Configuration Management System.
236        Copyright 1995-2011 Perforce Software.  All rights reserved.
237        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
238     """
239     global _p4_version_string
240     if not _p4_version_string:
241         a = p4_read_pipe_lines(["-V"])
242         _p4_version_string = a[-1].rstrip()
243     return _p4_version_string
244
245 def p4_integrate(src, dest):
246     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
247
248 def p4_sync(f, *options):
249     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
250
251 def p4_add(f):
252     # forcibly add file names with wildcards
253     if wildcard_present(f):
254         p4_system(["add", "-f", f])
255     else:
256         p4_system(["add", f])
257
258 def p4_delete(f):
259     p4_system(["delete", wildcard_encode(f)])
260
261 def p4_edit(f, *options):
262     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
263
264 def p4_revert(f):
265     p4_system(["revert", wildcard_encode(f)])
266
267 def p4_reopen(type, f):
268     p4_system(["reopen", "-t", type, wildcard_encode(f)])
269
270 def p4_reopen_in_change(changelist, files):
271     cmd = ["reopen", "-c", str(changelist)] + files
272     p4_system(cmd)
273
274 def p4_move(src, dest):
275     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
276
277 def p4_last_change():
278     results = p4CmdList(["changes", "-m", "1"])
279     return int(results[0]['change'])
280
281 def p4_describe(change):
282     """Make sure it returns a valid result by checking for
283        the presence of field "time".  Return a dict of the
284        results."""
285
286     ds = p4CmdList(["describe", "-s", str(change)])
287     if len(ds) != 1:
288         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
289
290     d = ds[0]
291
292     if "p4ExitCode" in d:
293         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
294                                                       str(d)))
295     if "code" in d:
296         if d["code"] == "error":
297             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
298
299     if "time" not in d:
300         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
301
302     return d
303
304 #
305 # Canonicalize the p4 type and return a tuple of the
306 # base type, plus any modifiers.  See "p4 help filetypes"
307 # for a list and explanation.
308 #
309 def split_p4_type(p4type):
310
311     p4_filetypes_historical = {
312         "ctempobj": "binary+Sw",
313         "ctext": "text+C",
314         "cxtext": "text+Cx",
315         "ktext": "text+k",
316         "kxtext": "text+kx",
317         "ltext": "text+F",
318         "tempobj": "binary+FSw",
319         "ubinary": "binary+F",
320         "uresource": "resource+F",
321         "uxbinary": "binary+Fx",
322         "xbinary": "binary+x",
323         "xltext": "text+Fx",
324         "xtempobj": "binary+Swx",
325         "xtext": "text+x",
326         "xunicode": "unicode+x",
327         "xutf16": "utf16+x",
328     }
329     if p4type in p4_filetypes_historical:
330         p4type = p4_filetypes_historical[p4type]
331     mods = ""
332     s = p4type.split("+")
333     base = s[0]
334     mods = ""
335     if len(s) > 1:
336         mods = s[1]
337     return (base, mods)
338
339 #
340 # return the raw p4 type of a file (text, text+ko, etc)
341 #
342 def p4_type(f):
343     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
344     return results[0]['headType']
345
346 #
347 # Given a type base and modifier, return a regexp matching
348 # the keywords that can be expanded in the file
349 #
350 def p4_keywords_regexp_for_type(base, type_mods):
351     if base in ("text", "unicode", "binary"):
352         kwords = None
353         if "ko" in type_mods:
354             kwords = 'Id|Header'
355         elif "k" in type_mods:
356             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
357         else:
358             return None
359         pattern = r"""
360             \$              # Starts with a dollar, followed by...
361             (%s)            # one of the keywords, followed by...
362             (:[^$\n]+)?     # possibly an old expansion, followed by...
363             \$              # another dollar
364             """ % kwords
365         return pattern
366     else:
367         return None
368
369 #
370 # Given a file, return a regexp matching the possible
371 # RCS keywords that will be expanded, or None for files
372 # with kw expansion turned off.
373 #
374 def p4_keywords_regexp_for_file(file):
375     if not os.path.exists(file):
376         return None
377     else:
378         (type_base, type_mods) = split_p4_type(p4_type(file))
379         return p4_keywords_regexp_for_type(type_base, type_mods)
380
381 def setP4ExecBit(file, mode):
382     # Reopens an already open file and changes the execute bit to match
383     # the execute bit setting in the passed in mode.
384
385     p4Type = "+x"
386
387     if not isModeExec(mode):
388         p4Type = getP4OpenedType(file)
389         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
390         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
391         if p4Type[-1] == "+":
392             p4Type = p4Type[0:-1]
393
394     p4_reopen(p4Type, file)
395
396 def getP4OpenedType(file):
397     # Returns the perforce file type for the given file.
398
399     result = p4_read_pipe(["opened", wildcard_encode(file)])
400     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
401     if match:
402         return match.group(1)
403     else:
404         die("Could not determine file type for %s (result: '%s')" % (file, result))
405
406 # Return the set of all p4 labels
407 def getP4Labels(depotPaths):
408     labels = set()
409     if isinstance(depotPaths,basestring):
410         depotPaths = [depotPaths]
411
412     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
413         label = l['label']
414         labels.add(label)
415
416     return labels
417
418 # Return the set of all git tags
419 def getGitTags():
420     gitTags = set()
421     for line in read_pipe_lines(["git", "tag"]):
422         tag = line.strip()
423         gitTags.add(tag)
424     return gitTags
425
426 def diffTreePattern():
427     # This is a simple generator for the diff tree regex pattern. This could be
428     # a class variable if this and parseDiffTreeEntry were a part of a class.
429     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
430     while True:
431         yield pattern
432
433 def parseDiffTreeEntry(entry):
434     """Parses a single diff tree entry into its component elements.
435
436     See git-diff-tree(1) manpage for details about the format of the diff
437     output. This method returns a dictionary with the following elements:
438
439     src_mode - The mode of the source file
440     dst_mode - The mode of the destination file
441     src_sha1 - The sha1 for the source file
442     dst_sha1 - The sha1 fr the destination file
443     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
444     status_score - The score for the status (applicable for 'C' and 'R'
445                    statuses). This is None if there is no score.
446     src - The path for the source file.
447     dst - The path for the destination file. This is only present for
448           copy or renames. If it is not present, this is None.
449
450     If the pattern is not matched, None is returned."""
451
452     match = diffTreePattern().next().match(entry)
453     if match:
454         return {
455             'src_mode': match.group(1),
456             'dst_mode': match.group(2),
457             'src_sha1': match.group(3),
458             'dst_sha1': match.group(4),
459             'status': match.group(5),
460             'status_score': match.group(6),
461             'src': match.group(7),
462             'dst': match.group(10)
463         }
464     return None
465
466 def isModeExec(mode):
467     # Returns True if the given git mode represents an executable file,
468     # otherwise False.
469     return mode[-3:] == "755"
470
471 def isModeExecChanged(src_mode, dst_mode):
472     return isModeExec(src_mode) != isModeExec(dst_mode)
473
474 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
475
476     if isinstance(cmd,basestring):
477         cmd = "-G " + cmd
478         expand = True
479     else:
480         cmd = ["-G"] + cmd
481         expand = False
482
483     cmd = p4_build_cmd(cmd)
484     if verbose:
485         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
486
487     # Use a temporary file to avoid deadlocks without
488     # subprocess.communicate(), which would put another copy
489     # of stdout into memory.
490     stdin_file = None
491     if stdin is not None:
492         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
493         if isinstance(stdin,basestring):
494             stdin_file.write(stdin)
495         else:
496             for i in stdin:
497                 stdin_file.write(i + '\n')
498         stdin_file.flush()
499         stdin_file.seek(0)
500
501     p4 = subprocess.Popen(cmd,
502                           shell=expand,
503                           stdin=stdin_file,
504                           stdout=subprocess.PIPE)
505
506     result = []
507     try:
508         while True:
509             entry = marshal.load(p4.stdout)
510             if cb is not None:
511                 cb(entry)
512             else:
513                 result.append(entry)
514     except EOFError:
515         pass
516     exitCode = p4.wait()
517     if exitCode != 0:
518         entry = {}
519         entry["p4ExitCode"] = exitCode
520         result.append(entry)
521
522     return result
523
524 def p4Cmd(cmd):
525     list = p4CmdList(cmd)
526     result = {}
527     for entry in list:
528         result.update(entry)
529     return result;
530
531 def p4Where(depotPath):
532     if not depotPath.endswith("/"):
533         depotPath += "/"
534     depotPathLong = depotPath + "..."
535     outputList = p4CmdList(["where", depotPathLong])
536     output = None
537     for entry in outputList:
538         if "depotFile" in entry:
539             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
540             # The base path always ends with "/...".
541             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
542                 output = entry
543                 break
544         elif "data" in entry:
545             data = entry.get("data")
546             space = data.find(" ")
547             if data[:space] == depotPath:
548                 output = entry
549                 break
550     if output == None:
551         return ""
552     if output["code"] == "error":
553         return ""
554     clientPath = ""
555     if "path" in output:
556         clientPath = output.get("path")
557     elif "data" in output:
558         data = output.get("data")
559         lastSpace = data.rfind(" ")
560         clientPath = data[lastSpace + 1:]
561
562     if clientPath.endswith("..."):
563         clientPath = clientPath[:-3]
564     return clientPath
565
566 def currentGitBranch():
567     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
568     if retcode != 0:
569         # on a detached head
570         return None
571     else:
572         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
573
574 def isValidGitDir(path):
575     if (os.path.exists(path + "/HEAD")
576         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
577         return True;
578     return False
579
580 def parseRevision(ref):
581     return read_pipe("git rev-parse %s" % ref).strip()
582
583 def branchExists(ref):
584     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
585                      ignore_error=True)
586     return len(rev) > 0
587
588 def extractLogMessageFromGitCommit(commit):
589     logMessage = ""
590
591     ## fixme: title is first line of commit, not 1st paragraph.
592     foundTitle = False
593     for log in read_pipe_lines("git cat-file commit %s" % commit):
594        if not foundTitle:
595            if len(log) == 1:
596                foundTitle = True
597            continue
598
599        logMessage += log
600     return logMessage
601
602 def extractSettingsGitLog(log):
603     values = {}
604     for line in log.split("\n"):
605         line = line.strip()
606         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
607         if not m:
608             continue
609
610         assignments = m.group(1).split (':')
611         for a in assignments:
612             vals = a.split ('=')
613             key = vals[0].strip()
614             val = ('='.join (vals[1:])).strip()
615             if val.endswith ('\"') and val.startswith('"'):
616                 val = val[1:-1]
617
618             values[key] = val
619
620     paths = values.get("depot-paths")
621     if not paths:
622         paths = values.get("depot-path")
623     if paths:
624         values['depot-paths'] = paths.split(',')
625     return values
626
627 def gitBranchExists(branch):
628     proc = subprocess.Popen(["git", "rev-parse", branch],
629                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
630     return proc.wait() == 0;
631
632 _gitConfig = {}
633
634 def gitConfig(key, typeSpecifier=None):
635     if not _gitConfig.has_key(key):
636         cmd = [ "git", "config" ]
637         if typeSpecifier:
638             cmd += [ typeSpecifier ]
639         cmd += [ key ]
640         s = read_pipe(cmd, ignore_error=True)
641         _gitConfig[key] = s.strip()
642     return _gitConfig[key]
643
644 def gitConfigBool(key):
645     """Return a bool, using git config --bool.  It is True only if the
646        variable is set to true, and False if set to false or not present
647        in the config."""
648
649     if not _gitConfig.has_key(key):
650         _gitConfig[key] = gitConfig(key, '--bool') == "true"
651     return _gitConfig[key]
652
653 def gitConfigInt(key):
654     if not _gitConfig.has_key(key):
655         cmd = [ "git", "config", "--int", key ]
656         s = read_pipe(cmd, ignore_error=True)
657         v = s.strip()
658         try:
659             _gitConfig[key] = int(gitConfig(key, '--int'))
660         except ValueError:
661             _gitConfig[key] = None
662     return _gitConfig[key]
663
664 def gitConfigList(key):
665     if not _gitConfig.has_key(key):
666         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
667         _gitConfig[key] = s.strip().split(os.linesep)
668         if _gitConfig[key] == ['']:
669             _gitConfig[key] = []
670     return _gitConfig[key]
671
672 def p4BranchesInGit(branchesAreInRemotes=True):
673     """Find all the branches whose names start with "p4/", looking
674        in remotes or heads as specified by the argument.  Return
675        a dictionary of { branch: revision } for each one found.
676        The branch names are the short names, without any
677        "p4/" prefix."""
678
679     branches = {}
680
681     cmdline = "git rev-parse --symbolic "
682     if branchesAreInRemotes:
683         cmdline += "--remotes"
684     else:
685         cmdline += "--branches"
686
687     for line in read_pipe_lines(cmdline):
688         line = line.strip()
689
690         # only import to p4/
691         if not line.startswith('p4/'):
692             continue
693         # special symbolic ref to p4/master
694         if line == "p4/HEAD":
695             continue
696
697         # strip off p4/ prefix
698         branch = line[len("p4/"):]
699
700         branches[branch] = parseRevision(line)
701
702     return branches
703
704 def branch_exists(branch):
705     """Make sure that the given ref name really exists."""
706
707     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
708     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
709     out, _ = p.communicate()
710     if p.returncode:
711         return False
712     # expect exactly one line of output: the branch name
713     return out.rstrip() == branch
714
715 def findUpstreamBranchPoint(head = "HEAD"):
716     branches = p4BranchesInGit()
717     # map from depot-path to branch name
718     branchByDepotPath = {}
719     for branch in branches.keys():
720         tip = branches[branch]
721         log = extractLogMessageFromGitCommit(tip)
722         settings = extractSettingsGitLog(log)
723         if settings.has_key("depot-paths"):
724             paths = ",".join(settings["depot-paths"])
725             branchByDepotPath[paths] = "remotes/p4/" + branch
726
727     settings = None
728     parent = 0
729     while parent < 65535:
730         commit = head + "~%s" % parent
731         log = extractLogMessageFromGitCommit(commit)
732         settings = extractSettingsGitLog(log)
733         if settings.has_key("depot-paths"):
734             paths = ",".join(settings["depot-paths"])
735             if branchByDepotPath.has_key(paths):
736                 return [branchByDepotPath[paths], settings]
737
738         parent = parent + 1
739
740     return ["", settings]
741
742 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
743     if not silent:
744         print ("Creating/updating branch(es) in %s based on origin branch(es)"
745                % localRefPrefix)
746
747     originPrefix = "origin/p4/"
748
749     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
750         line = line.strip()
751         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
752             continue
753
754         headName = line[len(originPrefix):]
755         remoteHead = localRefPrefix + headName
756         originHead = line
757
758         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
759         if (not original.has_key('depot-paths')
760             or not original.has_key('change')):
761             continue
762
763         update = False
764         if not gitBranchExists(remoteHead):
765             if verbose:
766                 print "creating %s" % remoteHead
767             update = True
768         else:
769             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
770             if settings.has_key('change') > 0:
771                 if settings['depot-paths'] == original['depot-paths']:
772                     originP4Change = int(original['change'])
773                     p4Change = int(settings['change'])
774                     if originP4Change > p4Change:
775                         print ("%s (%s) is newer than %s (%s). "
776                                "Updating p4 branch from origin."
777                                % (originHead, originP4Change,
778                                   remoteHead, p4Change))
779                         update = True
780                 else:
781                     print ("Ignoring: %s was imported from %s while "
782                            "%s was imported from %s"
783                            % (originHead, ','.join(original['depot-paths']),
784                               remoteHead, ','.join(settings['depot-paths'])))
785
786         if update:
787             system("git update-ref %s %s" % (remoteHead, originHead))
788
789 def originP4BranchesExist():
790         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
791
792
793 def p4ParseNumericChangeRange(parts):
794     changeStart = int(parts[0][1:])
795     if parts[1] == '#head':
796         changeEnd = p4_last_change()
797     else:
798         changeEnd = int(parts[1])
799
800     return (changeStart, changeEnd)
801
802 def chooseBlockSize(blockSize):
803     if blockSize:
804         return blockSize
805     else:
806         return defaultBlockSize
807
808 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
809     assert depotPaths
810
811     # Parse the change range into start and end. Try to find integer
812     # revision ranges as these can be broken up into blocks to avoid
813     # hitting server-side limits (maxrows, maxscanresults). But if
814     # that doesn't work, fall back to using the raw revision specifier
815     # strings, without using block mode.
816
817     if changeRange is None or changeRange == '':
818         changeStart = 1
819         changeEnd = p4_last_change()
820         block_size = chooseBlockSize(requestedBlockSize)
821     else:
822         parts = changeRange.split(',')
823         assert len(parts) == 2
824         try:
825             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
826             block_size = chooseBlockSize(requestedBlockSize)
827         except:
828             changeStart = parts[0][1:]
829             changeEnd = parts[1]
830             if requestedBlockSize:
831                 die("cannot use --changes-block-size with non-numeric revisions")
832             block_size = None
833
834     changes = []
835
836     # Retrieve changes a block at a time, to prevent running
837     # into a MaxResults/MaxScanRows error from the server.
838
839     while True:
840         cmd = ['changes']
841
842         if block_size:
843             end = min(changeEnd, changeStart + block_size)
844             revisionRange = "%d,%d" % (changeStart, end)
845         else:
846             revisionRange = "%s,%s" % (changeStart, changeEnd)
847
848         for p in depotPaths:
849             cmd += ["%s...@%s" % (p, revisionRange)]
850
851         # Insert changes in chronological order
852         for line in reversed(p4_read_pipe_lines(cmd)):
853             changes.append(int(line.split(" ")[1]))
854
855         if not block_size:
856             break
857
858         if end >= changeEnd:
859             break
860
861         changeStart = end + 1
862
863     changes = sorted(changes)
864     return changes
865
866 def p4PathStartsWith(path, prefix):
867     # This method tries to remedy a potential mixed-case issue:
868     #
869     # If UserA adds  //depot/DirA/file1
870     # and UserB adds //depot/dira/file2
871     #
872     # we may or may not have a problem. If you have core.ignorecase=true,
873     # we treat DirA and dira as the same directory
874     if gitConfigBool("core.ignorecase"):
875         return path.lower().startswith(prefix.lower())
876     return path.startswith(prefix)
877
878 def getClientSpec():
879     """Look at the p4 client spec, create a View() object that contains
880        all the mappings, and return it."""
881
882     specList = p4CmdList("client -o")
883     if len(specList) != 1:
884         die('Output from "client -o" is %d lines, expecting 1' %
885             len(specList))
886
887     # dictionary of all client parameters
888     entry = specList[0]
889
890     # the //client/ name
891     client_name = entry["Client"]
892
893     # just the keys that start with "View"
894     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
895
896     # hold this new View
897     view = View(client_name)
898
899     # append the lines, in order, to the view
900     for view_num in range(len(view_keys)):
901         k = "View%d" % view_num
902         if k not in view_keys:
903             die("Expected view key %s missing" % k)
904         view.append(entry[k])
905
906     return view
907
908 def getClientRoot():
909     """Grab the client directory."""
910
911     output = p4CmdList("client -o")
912     if len(output) != 1:
913         die('Output from "client -o" is %d lines, expecting 1' % len(output))
914
915     entry = output[0]
916     if "Root" not in entry:
917         die('Client has no "Root"')
918
919     return entry["Root"]
920
921 #
922 # P4 wildcards are not allowed in filenames.  P4 complains
923 # if you simply add them, but you can force it with "-f", in
924 # which case it translates them into %xx encoding internally.
925 #
926 def wildcard_decode(path):
927     # Search for and fix just these four characters.  Do % last so
928     # that fixing it does not inadvertently create new %-escapes.
929     # Cannot have * in a filename in windows; untested as to
930     # what p4 would do in such a case.
931     if not platform.system() == "Windows":
932         path = path.replace("%2A", "*")
933     path = path.replace("%23", "#") \
934                .replace("%40", "@") \
935                .replace("%25", "%")
936     return path
937
938 def wildcard_encode(path):
939     # do % first to avoid double-encoding the %s introduced here
940     path = path.replace("%", "%25") \
941                .replace("*", "%2A") \
942                .replace("#", "%23") \
943                .replace("@", "%40")
944     return path
945
946 def wildcard_present(path):
947     m = re.search("[*#@%]", path)
948     return m is not None
949
950 class LargeFileSystem(object):
951     """Base class for large file system support."""
952
953     def __init__(self, writeToGitStream):
954         self.largeFiles = set()
955         self.writeToGitStream = writeToGitStream
956
957     def generatePointer(self, cloneDestination, contentFile):
958         """Return the content of a pointer file that is stored in Git instead of
959            the actual content."""
960         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
961
962     def pushFile(self, localLargeFile):
963         """Push the actual content which is not stored in the Git repository to
964            a server."""
965         assert False, "Method 'pushFile' required in " + self.__class__.__name__
966
967     def hasLargeFileExtension(self, relPath):
968         return reduce(
969             lambda a, b: a or b,
970             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
971             False
972         )
973
974     def generateTempFile(self, contents):
975         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
976         for d in contents:
977             contentFile.write(d)
978         contentFile.close()
979         return contentFile.name
980
981     def exceedsLargeFileThreshold(self, relPath, contents):
982         if gitConfigInt('git-p4.largeFileThreshold'):
983             contentsSize = sum(len(d) for d in contents)
984             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
985                 return True
986         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
987             contentsSize = sum(len(d) for d in contents)
988             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
989                 return False
990             contentTempFile = self.generateTempFile(contents)
991             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
992             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
993             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
994             zf.close()
995             compressedContentsSize = zf.infolist()[0].compress_size
996             os.remove(contentTempFile)
997             os.remove(compressedContentFile.name)
998             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
999                 return True
1000         return False
1001
1002     def addLargeFile(self, relPath):
1003         self.largeFiles.add(relPath)
1004
1005     def removeLargeFile(self, relPath):
1006         self.largeFiles.remove(relPath)
1007
1008     def isLargeFile(self, relPath):
1009         return relPath in self.largeFiles
1010
1011     def processContent(self, git_mode, relPath, contents):
1012         """Processes the content of git fast import. This method decides if a
1013            file is stored in the large file system and handles all necessary
1014            steps."""
1015         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1016             contentTempFile = self.generateTempFile(contents)
1017             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1018             if pointer_git_mode:
1019                 git_mode = pointer_git_mode
1020             if localLargeFile:
1021                 # Move temp file to final location in large file system
1022                 largeFileDir = os.path.dirname(localLargeFile)
1023                 if not os.path.isdir(largeFileDir):
1024                     os.makedirs(largeFileDir)
1025                 shutil.move(contentTempFile, localLargeFile)
1026                 self.addLargeFile(relPath)
1027                 if gitConfigBool('git-p4.largeFilePush'):
1028                     self.pushFile(localLargeFile)
1029                 if verbose:
1030                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1031         return (git_mode, contents)
1032
1033 class MockLFS(LargeFileSystem):
1034     """Mock large file system for testing."""
1035
1036     def generatePointer(self, contentFile):
1037         """The pointer content is the original content prefixed with "pointer-".
1038            The local filename of the large file storage is derived from the file content.
1039            """
1040         with open(contentFile, 'r') as f:
1041             content = next(f)
1042             gitMode = '100644'
1043             pointerContents = 'pointer-' + content
1044             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1045             return (gitMode, pointerContents, localLargeFile)
1046
1047     def pushFile(self, localLargeFile):
1048         """The remote filename of the large file storage is the same as the local
1049            one but in a different directory.
1050            """
1051         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1052         if not os.path.exists(remotePath):
1053             os.makedirs(remotePath)
1054         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1055
1056 class GitLFS(LargeFileSystem):
1057     """Git LFS as backend for the git-p4 large file system.
1058        See https://git-lfs.github.com/ for details."""
1059
1060     def __init__(self, *args):
1061         LargeFileSystem.__init__(self, *args)
1062         self.baseGitAttributes = []
1063
1064     def generatePointer(self, contentFile):
1065         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1066            mode and content which is stored in the Git repository instead of
1067            the actual content. Return also the new location of the actual
1068            content.
1069            """
1070         if os.path.getsize(contentFile) == 0:
1071             return (None, '', None)
1072
1073         pointerProcess = subprocess.Popen(
1074             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1075             stdout=subprocess.PIPE
1076         )
1077         pointerFile = pointerProcess.stdout.read()
1078         if pointerProcess.wait():
1079             os.remove(contentFile)
1080             die('git-lfs pointer command failed. Did you install the extension?')
1081
1082         # Git LFS removed the preamble in the output of the 'pointer' command
1083         # starting from version 1.2.0. Check for the preamble here to support
1084         # earlier versions.
1085         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1086         if pointerFile.startswith('Git LFS pointer for'):
1087             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1088
1089         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1090         localLargeFile = os.path.join(
1091             os.getcwd(),
1092             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1093             oid,
1094         )
1095         # LFS Spec states that pointer files should not have the executable bit set.
1096         gitMode = '100644'
1097         return (gitMode, pointerFile, localLargeFile)
1098
1099     def pushFile(self, localLargeFile):
1100         uploadProcess = subprocess.Popen(
1101             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1102         )
1103         if uploadProcess.wait():
1104             die('git-lfs push command failed. Did you define a remote?')
1105
1106     def generateGitAttributes(self):
1107         return (
1108             self.baseGitAttributes +
1109             [
1110                 '\n',
1111                 '#\n',
1112                 '# Git LFS (see https://git-lfs.github.com/)\n',
1113                 '#\n',
1114             ] +
1115             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1116                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1117             ] +
1118             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1119                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1120             ]
1121         )
1122
1123     def addLargeFile(self, relPath):
1124         LargeFileSystem.addLargeFile(self, relPath)
1125         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1126
1127     def removeLargeFile(self, relPath):
1128         LargeFileSystem.removeLargeFile(self, relPath)
1129         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1130
1131     def processContent(self, git_mode, relPath, contents):
1132         if relPath == '.gitattributes':
1133             self.baseGitAttributes = contents
1134             return (git_mode, self.generateGitAttributes())
1135         else:
1136             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1137
1138 class Command:
1139     def __init__(self):
1140         self.usage = "usage: %prog [options]"
1141         self.needsGit = True
1142         self.verbose = False
1143
1144 class P4UserMap:
1145     def __init__(self):
1146         self.userMapFromPerforceServer = False
1147         self.myP4UserId = None
1148
1149     def p4UserId(self):
1150         if self.myP4UserId:
1151             return self.myP4UserId
1152
1153         results = p4CmdList("user -o")
1154         for r in results:
1155             if r.has_key('User'):
1156                 self.myP4UserId = r['User']
1157                 return r['User']
1158         die("Could not find your p4 user id")
1159
1160     def p4UserIsMe(self, p4User):
1161         # return True if the given p4 user is actually me
1162         me = self.p4UserId()
1163         if not p4User or p4User != me:
1164             return False
1165         else:
1166             return True
1167
1168     def getUserCacheFilename(self):
1169         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1170         return home + "/.gitp4-usercache.txt"
1171
1172     def getUserMapFromPerforceServer(self):
1173         if self.userMapFromPerforceServer:
1174             return
1175         self.users = {}
1176         self.emails = {}
1177
1178         for output in p4CmdList("users"):
1179             if not output.has_key("User"):
1180                 continue
1181             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1182             self.emails[output["Email"]] = output["User"]
1183
1184         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1185         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1186             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1187             if mapUser and len(mapUser[0]) == 3:
1188                 user = mapUser[0][0]
1189                 fullname = mapUser[0][1]
1190                 email = mapUser[0][2]
1191                 self.users[user] = fullname + " <" + email + ">"
1192                 self.emails[email] = user
1193
1194         s = ''
1195         for (key, val) in self.users.items():
1196             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1197
1198         open(self.getUserCacheFilename(), "wb").write(s)
1199         self.userMapFromPerforceServer = True
1200
1201     def loadUserMapFromCache(self):
1202         self.users = {}
1203         self.userMapFromPerforceServer = False
1204         try:
1205             cache = open(self.getUserCacheFilename(), "rb")
1206             lines = cache.readlines()
1207             cache.close()
1208             for line in lines:
1209                 entry = line.strip().split("\t")
1210                 self.users[entry[0]] = entry[1]
1211         except IOError:
1212             self.getUserMapFromPerforceServer()
1213
1214 class P4Debug(Command):
1215     def __init__(self):
1216         Command.__init__(self)
1217         self.options = []
1218         self.description = "A tool to debug the output of p4 -G."
1219         self.needsGit = False
1220
1221     def run(self, args):
1222         j = 0
1223         for output in p4CmdList(args):
1224             print 'Element: %d' % j
1225             j += 1
1226             print output
1227         return True
1228
1229 class P4RollBack(Command):
1230     def __init__(self):
1231         Command.__init__(self)
1232         self.options = [
1233             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1234         ]
1235         self.description = "A tool to debug the multi-branch import. Don't use :)"
1236         self.rollbackLocalBranches = False
1237
1238     def run(self, args):
1239         if len(args) != 1:
1240             return False
1241         maxChange = int(args[0])
1242
1243         if "p4ExitCode" in p4Cmd("changes -m 1"):
1244             die("Problems executing p4");
1245
1246         if self.rollbackLocalBranches:
1247             refPrefix = "refs/heads/"
1248             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1249         else:
1250             refPrefix = "refs/remotes/"
1251             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1252
1253         for line in lines:
1254             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1255                 line = line.strip()
1256                 ref = refPrefix + line
1257                 log = extractLogMessageFromGitCommit(ref)
1258                 settings = extractSettingsGitLog(log)
1259
1260                 depotPaths = settings['depot-paths']
1261                 change = settings['change']
1262
1263                 changed = False
1264
1265                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1266                                                            for p in depotPaths]))) == 0:
1267                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1268                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1269                     continue
1270
1271                 while change and int(change) > maxChange:
1272                     changed = True
1273                     if self.verbose:
1274                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1275                     system("git update-ref %s \"%s^\"" % (ref, ref))
1276                     log = extractLogMessageFromGitCommit(ref)
1277                     settings =  extractSettingsGitLog(log)
1278
1279
1280                     depotPaths = settings['depot-paths']
1281                     change = settings['change']
1282
1283                 if changed:
1284                     print "%s rewound to %s" % (ref, change)
1285
1286         return True
1287
1288 class P4Submit(Command, P4UserMap):
1289
1290     conflict_behavior_choices = ("ask", "skip", "quit")
1291
1292     def __init__(self):
1293         Command.__init__(self)
1294         P4UserMap.__init__(self)
1295         self.options = [
1296                 optparse.make_option("--origin", dest="origin"),
1297                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1298                 # preserve the user, requires relevant p4 permissions
1299                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1300                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1301                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1302                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1303                 optparse.make_option("--conflict", dest="conflict_behavior",
1304                                      choices=self.conflict_behavior_choices),
1305                 optparse.make_option("--branch", dest="branch"),
1306                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1307                                      help="Shelve instead of submit. Shelved files are reverted, "
1308                                      "restoring the workspace to the state before the shelve"),
1309                 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1310                                      metavar="CHANGELIST",
1311                                      help="update an existing shelved changelist, implies --shelve")
1312         ]
1313         self.description = "Submit changes from git to the perforce depot."
1314         self.usage += " [name of git branch to submit into perforce depot]"
1315         self.origin = ""
1316         self.detectRenames = False
1317         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1318         self.dry_run = False
1319         self.shelve = False
1320         self.update_shelve = None
1321         self.prepare_p4_only = False
1322         self.conflict_behavior = None
1323         self.isWindows = (platform.system() == "Windows")
1324         self.exportLabels = False
1325         self.p4HasMoveCommand = p4_has_move_command()
1326         self.branch = None
1327
1328         if gitConfig('git-p4.largeFileSystem'):
1329             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1330
1331     def check(self):
1332         if len(p4CmdList("opened ...")) > 0:
1333             die("You have files opened with perforce! Close them before starting the sync.")
1334
1335     def separate_jobs_from_description(self, message):
1336         """Extract and return a possible Jobs field in the commit
1337            message.  It goes into a separate section in the p4 change
1338            specification.
1339
1340            A jobs line starts with "Jobs:" and looks like a new field
1341            in a form.  Values are white-space separated on the same
1342            line or on following lines that start with a tab.
1343
1344            This does not parse and extract the full git commit message
1345            like a p4 form.  It just sees the Jobs: line as a marker
1346            to pass everything from then on directly into the p4 form,
1347            but outside the description section.
1348
1349            Return a tuple (stripped log message, jobs string)."""
1350
1351         m = re.search(r'^Jobs:', message, re.MULTILINE)
1352         if m is None:
1353             return (message, None)
1354
1355         jobtext = message[m.start():]
1356         stripped_message = message[:m.start()].rstrip()
1357         return (stripped_message, jobtext)
1358
1359     def prepareLogMessage(self, template, message, jobs):
1360         """Edits the template returned from "p4 change -o" to insert
1361            the message in the Description field, and the jobs text in
1362            the Jobs field."""
1363         result = ""
1364
1365         inDescriptionSection = False
1366
1367         for line in template.split("\n"):
1368             if line.startswith("#"):
1369                 result += line + "\n"
1370                 continue
1371
1372             if inDescriptionSection:
1373                 if line.startswith("Files:") or line.startswith("Jobs:"):
1374                     inDescriptionSection = False
1375                     # insert Jobs section
1376                     if jobs:
1377                         result += jobs + "\n"
1378                 else:
1379                     continue
1380             else:
1381                 if line.startswith("Description:"):
1382                     inDescriptionSection = True
1383                     line += "\n"
1384                     for messageLine in message.split("\n"):
1385                         line += "\t" + messageLine + "\n"
1386
1387             result += line + "\n"
1388
1389         return result
1390
1391     def patchRCSKeywords(self, file, pattern):
1392         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1393         (handle, outFileName) = tempfile.mkstemp(dir='.')
1394         try:
1395             outFile = os.fdopen(handle, "w+")
1396             inFile = open(file, "r")
1397             regexp = re.compile(pattern, re.VERBOSE)
1398             for line in inFile.readlines():
1399                 line = regexp.sub(r'$\1$', line)
1400                 outFile.write(line)
1401             inFile.close()
1402             outFile.close()
1403             # Forcibly overwrite the original file
1404             os.unlink(file)
1405             shutil.move(outFileName, file)
1406         except:
1407             # cleanup our temporary file
1408             os.unlink(outFileName)
1409             print "Failed to strip RCS keywords in %s" % file
1410             raise
1411
1412         print "Patched up RCS keywords in %s" % file
1413
1414     def p4UserForCommit(self,id):
1415         # Return the tuple (perforce user,git email) for a given git commit id
1416         self.getUserMapFromPerforceServer()
1417         gitEmail = read_pipe(["git", "log", "--max-count=1",
1418                               "--format=%ae", id])
1419         gitEmail = gitEmail.strip()
1420         if not self.emails.has_key(gitEmail):
1421             return (None,gitEmail)
1422         else:
1423             return (self.emails[gitEmail],gitEmail)
1424
1425     def checkValidP4Users(self,commits):
1426         # check if any git authors cannot be mapped to p4 users
1427         for id in commits:
1428             (user,email) = self.p4UserForCommit(id)
1429             if not user:
1430                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1431                 if gitConfigBool("git-p4.allowMissingP4Users"):
1432                     print "%s" % msg
1433                 else:
1434                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1435
1436     def lastP4Changelist(self):
1437         # Get back the last changelist number submitted in this client spec. This
1438         # then gets used to patch up the username in the change. If the same
1439         # client spec is being used by multiple processes then this might go
1440         # wrong.
1441         results = p4CmdList("client -o")        # find the current client
1442         client = None
1443         for r in results:
1444             if r.has_key('Client'):
1445                 client = r['Client']
1446                 break
1447         if not client:
1448             die("could not get client spec")
1449         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1450         for r in results:
1451             if r.has_key('change'):
1452                 return r['change']
1453         die("Could not get changelist number for last submit - cannot patch up user details")
1454
1455     def modifyChangelistUser(self, changelist, newUser):
1456         # fixup the user field of a changelist after it has been submitted.
1457         changes = p4CmdList("change -o %s" % changelist)
1458         if len(changes) != 1:
1459             die("Bad output from p4 change modifying %s to user %s" %
1460                 (changelist, newUser))
1461
1462         c = changes[0]
1463         if c['User'] == newUser: return   # nothing to do
1464         c['User'] = newUser
1465         input = marshal.dumps(c)
1466
1467         result = p4CmdList("change -f -i", stdin=input)
1468         for r in result:
1469             if r.has_key('code'):
1470                 if r['code'] == 'error':
1471                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1472             if r.has_key('data'):
1473                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1474                 return
1475         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1476
1477     def canChangeChangelists(self):
1478         # check to see if we have p4 admin or super-user permissions, either of
1479         # which are required to modify changelists.
1480         results = p4CmdList(["protects", self.depotPath])
1481         for r in results:
1482             if r.has_key('perm'):
1483                 if r['perm'] == 'admin':
1484                     return 1
1485                 if r['perm'] == 'super':
1486                     return 1
1487         return 0
1488
1489     def prepareSubmitTemplate(self, changelist=None):
1490         """Run "p4 change -o" to grab a change specification template.
1491            This does not use "p4 -G", as it is nice to keep the submission
1492            template in original order, since a human might edit it.
1493
1494            Remove lines in the Files section that show changes to files
1495            outside the depot path we're committing into."""
1496
1497         [upstream, settings] = findUpstreamBranchPoint()
1498
1499         template = ""
1500         inFilesSection = False
1501         args = ['change', '-o']
1502         if changelist:
1503             args.append(str(changelist))
1504
1505         for line in p4_read_pipe_lines(args):
1506             if line.endswith("\r\n"):
1507                 line = line[:-2] + "\n"
1508             if inFilesSection:
1509                 if line.startswith("\t"):
1510                     # path starts and ends with a tab
1511                     path = line[1:]
1512                     lastTab = path.rfind("\t")
1513                     if lastTab != -1:
1514                         path = path[:lastTab]
1515                         if settings.has_key('depot-paths'):
1516                             if not [p for p in settings['depot-paths']
1517                                     if p4PathStartsWith(path, p)]:
1518                                 continue
1519                         else:
1520                             if not p4PathStartsWith(path, self.depotPath):
1521                                 continue
1522                 else:
1523                     inFilesSection = False
1524             else:
1525                 if line.startswith("Files:"):
1526                     inFilesSection = True
1527
1528             template += line
1529
1530         return template
1531
1532     def edit_template(self, template_file):
1533         """Invoke the editor to let the user change the submission
1534            message.  Return true if okay to continue with the submit."""
1535
1536         # if configured to skip the editing part, just submit
1537         if gitConfigBool("git-p4.skipSubmitEdit"):
1538             return True
1539
1540         # look at the modification time, to check later if the user saved
1541         # the file
1542         mtime = os.stat(template_file).st_mtime
1543
1544         # invoke the editor
1545         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1546             editor = os.environ.get("P4EDITOR")
1547         else:
1548             editor = read_pipe("git var GIT_EDITOR").strip()
1549         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1550
1551         # If the file was not saved, prompt to see if this patch should
1552         # be skipped.  But skip this verification step if configured so.
1553         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1554             return True
1555
1556         # modification time updated means user saved the file
1557         if os.stat(template_file).st_mtime > mtime:
1558             return True
1559
1560         while True:
1561             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1562             if response == 'y':
1563                 return True
1564             if response == 'n':
1565                 return False
1566
1567     def get_diff_description(self, editedFiles, filesToAdd):
1568         # diff
1569         if os.environ.has_key("P4DIFF"):
1570             del(os.environ["P4DIFF"])
1571         diff = ""
1572         for editedFile in editedFiles:
1573             diff += p4_read_pipe(['diff', '-du',
1574                                   wildcard_encode(editedFile)])
1575
1576         # new file diff
1577         newdiff = ""
1578         for newFile in filesToAdd:
1579             newdiff += "==== new file ====\n"
1580             newdiff += "--- /dev/null\n"
1581             newdiff += "+++ %s\n" % newFile
1582             f = open(newFile, "r")
1583             for line in f.readlines():
1584                 newdiff += "+" + line
1585             f.close()
1586
1587         return (diff + newdiff).replace('\r\n', '\n')
1588
1589     def applyCommit(self, id):
1590         """Apply one commit, return True if it succeeded."""
1591
1592         print "Applying", read_pipe(["git", "show", "-s",
1593                                      "--format=format:%h %s", id])
1594
1595         (p4User, gitEmail) = self.p4UserForCommit(id)
1596
1597         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1598         filesToAdd = set()
1599         filesToChangeType = set()
1600         filesToDelete = set()
1601         editedFiles = set()
1602         pureRenameCopy = set()
1603         filesToChangeExecBit = {}
1604         all_files = list()
1605
1606         for line in diff:
1607             diff = parseDiffTreeEntry(line)
1608             modifier = diff['status']
1609             path = diff['src']
1610             all_files.append(path)
1611
1612             if modifier == "M":
1613                 p4_edit(path)
1614                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1615                     filesToChangeExecBit[path] = diff['dst_mode']
1616                 editedFiles.add(path)
1617             elif modifier == "A":
1618                 filesToAdd.add(path)
1619                 filesToChangeExecBit[path] = diff['dst_mode']
1620                 if path in filesToDelete:
1621                     filesToDelete.remove(path)
1622             elif modifier == "D":
1623                 filesToDelete.add(path)
1624                 if path in filesToAdd:
1625                     filesToAdd.remove(path)
1626             elif modifier == "C":
1627                 src, dest = diff['src'], diff['dst']
1628                 p4_integrate(src, dest)
1629                 pureRenameCopy.add(dest)
1630                 if diff['src_sha1'] != diff['dst_sha1']:
1631                     p4_edit(dest)
1632                     pureRenameCopy.discard(dest)
1633                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1634                     p4_edit(dest)
1635                     pureRenameCopy.discard(dest)
1636                     filesToChangeExecBit[dest] = diff['dst_mode']
1637                 if self.isWindows:
1638                     # turn off read-only attribute
1639                     os.chmod(dest, stat.S_IWRITE)
1640                 os.unlink(dest)
1641                 editedFiles.add(dest)
1642             elif modifier == "R":
1643                 src, dest = diff['src'], diff['dst']
1644                 if self.p4HasMoveCommand:
1645                     p4_edit(src)        # src must be open before move
1646                     p4_move(src, dest)  # opens for (move/delete, move/add)
1647                 else:
1648                     p4_integrate(src, dest)
1649                     if diff['src_sha1'] != diff['dst_sha1']:
1650                         p4_edit(dest)
1651                     else:
1652                         pureRenameCopy.add(dest)
1653                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1654                     if not self.p4HasMoveCommand:
1655                         p4_edit(dest)   # with move: already open, writable
1656                     filesToChangeExecBit[dest] = diff['dst_mode']
1657                 if not self.p4HasMoveCommand:
1658                     if self.isWindows:
1659                         os.chmod(dest, stat.S_IWRITE)
1660                     os.unlink(dest)
1661                     filesToDelete.add(src)
1662                 editedFiles.add(dest)
1663             elif modifier == "T":
1664                 filesToChangeType.add(path)
1665             else:
1666                 die("unknown modifier %s for %s" % (modifier, path))
1667
1668         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1669         patchcmd = diffcmd + " | git apply "
1670         tryPatchCmd = patchcmd + "--check -"
1671         applyPatchCmd = patchcmd + "--check --apply -"
1672         patch_succeeded = True
1673
1674         if os.system(tryPatchCmd) != 0:
1675             fixed_rcs_keywords = False
1676             patch_succeeded = False
1677             print "Unfortunately applying the change failed!"
1678
1679             # Patch failed, maybe it's just RCS keyword woes. Look through
1680             # the patch to see if that's possible.
1681             if gitConfigBool("git-p4.attemptRCSCleanup"):
1682                 file = None
1683                 pattern = None
1684                 kwfiles = {}
1685                 for file in editedFiles | filesToDelete:
1686                     # did this file's delta contain RCS keywords?
1687                     pattern = p4_keywords_regexp_for_file(file)
1688
1689                     if pattern:
1690                         # this file is a possibility...look for RCS keywords.
1691                         regexp = re.compile(pattern, re.VERBOSE)
1692                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1693                             if regexp.search(line):
1694                                 if verbose:
1695                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1696                                 kwfiles[file] = pattern
1697                                 break
1698
1699                 for file in kwfiles:
1700                     if verbose:
1701                         print "zapping %s with %s" % (line,pattern)
1702                     # File is being deleted, so not open in p4.  Must
1703                     # disable the read-only bit on windows.
1704                     if self.isWindows and file not in editedFiles:
1705                         os.chmod(file, stat.S_IWRITE)
1706                     self.patchRCSKeywords(file, kwfiles[file])
1707                     fixed_rcs_keywords = True
1708
1709             if fixed_rcs_keywords:
1710                 print "Retrying the patch with RCS keywords cleaned up"
1711                 if os.system(tryPatchCmd) == 0:
1712                     patch_succeeded = True
1713
1714         if not patch_succeeded:
1715             for f in editedFiles:
1716                 p4_revert(f)
1717             return False
1718
1719         #
1720         # Apply the patch for real, and do add/delete/+x handling.
1721         #
1722         system(applyPatchCmd)
1723
1724         for f in filesToChangeType:
1725             p4_edit(f, "-t", "auto")
1726         for f in filesToAdd:
1727             p4_add(f)
1728         for f in filesToDelete:
1729             p4_revert(f)
1730             p4_delete(f)
1731
1732         # Set/clear executable bits
1733         for f in filesToChangeExecBit.keys():
1734             mode = filesToChangeExecBit[f]
1735             setP4ExecBit(f, mode)
1736
1737         if self.update_shelve:
1738             print("all_files = %s" % str(all_files))
1739             p4_reopen_in_change(self.update_shelve, all_files)
1740
1741         #
1742         # Build p4 change description, starting with the contents
1743         # of the git commit message.
1744         #
1745         logMessage = extractLogMessageFromGitCommit(id)
1746         logMessage = logMessage.strip()
1747         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1748
1749         template = self.prepareSubmitTemplate(self.update_shelve)
1750         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1751
1752         if self.preserveUser:
1753            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1754
1755         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1756             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1757             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1758             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1759
1760         separatorLine = "######## everything below this line is just the diff #######\n"
1761         if not self.prepare_p4_only:
1762             submitTemplate += separatorLine
1763             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1764
1765         (handle, fileName) = tempfile.mkstemp()
1766         tmpFile = os.fdopen(handle, "w+b")
1767         if self.isWindows:
1768             submitTemplate = submitTemplate.replace("\n", "\r\n")
1769         tmpFile.write(submitTemplate)
1770         tmpFile.close()
1771
1772         if self.prepare_p4_only:
1773             #
1774             # Leave the p4 tree prepared, and the submit template around
1775             # and let the user decide what to do next
1776             #
1777             print
1778             print "P4 workspace prepared for submission."
1779             print "To submit or revert, go to client workspace"
1780             print "  " + self.clientPath
1781             print
1782             print "To submit, use \"p4 submit\" to write a new description,"
1783             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1784                   " \"git p4\"." % fileName
1785             print "You can delete the file \"%s\" when finished." % fileName
1786
1787             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1788                 print "To preserve change ownership by user %s, you must\n" \
1789                       "do \"p4 change -f <change>\" after submitting and\n" \
1790                       "edit the User field."
1791             if pureRenameCopy:
1792                 print "After submitting, renamed files must be re-synced."
1793                 print "Invoke \"p4 sync -f\" on each of these files:"
1794                 for f in pureRenameCopy:
1795                     print "  " + f
1796
1797             print
1798             print "To revert the changes, use \"p4 revert ...\", and delete"
1799             print "the submit template file \"%s\"" % fileName
1800             if filesToAdd:
1801                 print "Since the commit adds new files, they must be deleted:"
1802                 for f in filesToAdd:
1803                     print "  " + f
1804             print
1805             return True
1806
1807         #
1808         # Let the user edit the change description, then submit it.
1809         #
1810         submitted = False
1811
1812         try:
1813             if self.edit_template(fileName):
1814                 # read the edited message and submit
1815                 tmpFile = open(fileName, "rb")
1816                 message = tmpFile.read()
1817                 tmpFile.close()
1818                 if self.isWindows:
1819                     message = message.replace("\r\n", "\n")
1820                 submitTemplate = message[:message.index(separatorLine)]
1821
1822                 if self.update_shelve:
1823                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1824                 elif self.shelve:
1825                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1826                 else:
1827                     p4_write_pipe(['submit', '-i'], submitTemplate)
1828                     # The rename/copy happened by applying a patch that created a
1829                     # new file.  This leaves it writable, which confuses p4.
1830                     for f in pureRenameCopy:
1831                         p4_sync(f, "-f")
1832
1833                 if self.preserveUser:
1834                     if p4User:
1835                         # Get last changelist number. Cannot easily get it from
1836                         # the submit command output as the output is
1837                         # unmarshalled.
1838                         changelist = self.lastP4Changelist()
1839                         self.modifyChangelistUser(changelist, p4User)
1840
1841                 submitted = True
1842
1843         finally:
1844             # skip this patch
1845             if not submitted or self.shelve:
1846                 if self.shelve:
1847                     print ("Reverting shelved files.")
1848                 else:
1849                     print ("Submission cancelled, undoing p4 changes.")
1850                 for f in editedFiles | filesToDelete:
1851                     p4_revert(f)
1852                 for f in filesToAdd:
1853                     p4_revert(f)
1854                     os.remove(f)
1855
1856         os.remove(fileName)
1857         return submitted
1858
1859     # Export git tags as p4 labels. Create a p4 label and then tag
1860     # with that.
1861     def exportGitTags(self, gitTags):
1862         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1863         if len(validLabelRegexp) == 0:
1864             validLabelRegexp = defaultLabelRegexp
1865         m = re.compile(validLabelRegexp)
1866
1867         for name in gitTags:
1868
1869             if not m.match(name):
1870                 if verbose:
1871                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1872                 continue
1873
1874             # Get the p4 commit this corresponds to
1875             logMessage = extractLogMessageFromGitCommit(name)
1876             values = extractSettingsGitLog(logMessage)
1877
1878             if not values.has_key('change'):
1879                 # a tag pointing to something not sent to p4; ignore
1880                 if verbose:
1881                     print "git tag %s does not give a p4 commit" % name
1882                 continue
1883             else:
1884                 changelist = values['change']
1885
1886             # Get the tag details.
1887             inHeader = True
1888             isAnnotated = False
1889             body = []
1890             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1891                 l = l.strip()
1892                 if inHeader:
1893                     if re.match(r'tag\s+', l):
1894                         isAnnotated = True
1895                     elif re.match(r'\s*$', l):
1896                         inHeader = False
1897                         continue
1898                 else:
1899                     body.append(l)
1900
1901             if not isAnnotated:
1902                 body = ["lightweight tag imported by git p4\n"]
1903
1904             # Create the label - use the same view as the client spec we are using
1905             clientSpec = getClientSpec()
1906
1907             labelTemplate  = "Label: %s\n" % name
1908             labelTemplate += "Description:\n"
1909             for b in body:
1910                 labelTemplate += "\t" + b + "\n"
1911             labelTemplate += "View:\n"
1912             for depot_side in clientSpec.mappings:
1913                 labelTemplate += "\t%s\n" % depot_side
1914
1915             if self.dry_run:
1916                 print "Would create p4 label %s for tag" % name
1917             elif self.prepare_p4_only:
1918                 print "Not creating p4 label %s for tag due to option" \
1919                       " --prepare-p4-only" % name
1920             else:
1921                 p4_write_pipe(["label", "-i"], labelTemplate)
1922
1923                 # Use the label
1924                 p4_system(["tag", "-l", name] +
1925                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1926
1927                 if verbose:
1928                     print "created p4 label for tag %s" % name
1929
1930     def run(self, args):
1931         if len(args) == 0:
1932             self.master = currentGitBranch()
1933         elif len(args) == 1:
1934             self.master = args[0]
1935             if not branchExists(self.master):
1936                 die("Branch %s does not exist" % self.master)
1937         else:
1938             return False
1939
1940         if self.master:
1941             allowSubmit = gitConfig("git-p4.allowSubmit")
1942             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1943                 die("%s is not in git-p4.allowSubmit" % self.master)
1944
1945         [upstream, settings] = findUpstreamBranchPoint()
1946         self.depotPath = settings['depot-paths'][0]
1947         if len(self.origin) == 0:
1948             self.origin = upstream
1949
1950         if self.update_shelve:
1951             self.shelve = True
1952
1953         if self.preserveUser:
1954             if not self.canChangeChangelists():
1955                 die("Cannot preserve user names without p4 super-user or admin permissions")
1956
1957         # if not set from the command line, try the config file
1958         if self.conflict_behavior is None:
1959             val = gitConfig("git-p4.conflict")
1960             if val:
1961                 if val not in self.conflict_behavior_choices:
1962                     die("Invalid value '%s' for config git-p4.conflict" % val)
1963             else:
1964                 val = "ask"
1965             self.conflict_behavior = val
1966
1967         if self.verbose:
1968             print "Origin branch is " + self.origin
1969
1970         if len(self.depotPath) == 0:
1971             print "Internal error: cannot locate perforce depot path from existing branches"
1972             sys.exit(128)
1973
1974         self.useClientSpec = False
1975         if gitConfigBool("git-p4.useclientspec"):
1976             self.useClientSpec = True
1977         if self.useClientSpec:
1978             self.clientSpecDirs = getClientSpec()
1979
1980         # Check for the existence of P4 branches
1981         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1982
1983         if self.useClientSpec and not branchesDetected:
1984             # all files are relative to the client spec
1985             self.clientPath = getClientRoot()
1986         else:
1987             self.clientPath = p4Where(self.depotPath)
1988
1989         if self.clientPath == "":
1990             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1991
1992         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1993         self.oldWorkingDirectory = os.getcwd()
1994
1995         # ensure the clientPath exists
1996         new_client_dir = False
1997         if not os.path.exists(self.clientPath):
1998             new_client_dir = True
1999             os.makedirs(self.clientPath)
2000
2001         chdir(self.clientPath, is_client_path=True)
2002         if self.dry_run:
2003             print "Would synchronize p4 checkout in %s" % self.clientPath
2004         else:
2005             print "Synchronizing p4 checkout..."
2006             if new_client_dir:
2007                 # old one was destroyed, and maybe nobody told p4
2008                 p4_sync("...", "-f")
2009             else:
2010                 p4_sync("...")
2011         self.check()
2012
2013         commits = []
2014         if self.master:
2015             commitish = self.master
2016         else:
2017             commitish = 'HEAD'
2018
2019         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2020             commits.append(line.strip())
2021         commits.reverse()
2022
2023         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2024             self.checkAuthorship = False
2025         else:
2026             self.checkAuthorship = True
2027
2028         if self.preserveUser:
2029             self.checkValidP4Users(commits)
2030
2031         #
2032         # Build up a set of options to be passed to diff when
2033         # submitting each commit to p4.
2034         #
2035         if self.detectRenames:
2036             # command-line -M arg
2037             self.diffOpts = "-M"
2038         else:
2039             # If not explicitly set check the config variable
2040             detectRenames = gitConfig("git-p4.detectRenames")
2041
2042             if detectRenames.lower() == "false" or detectRenames == "":
2043                 self.diffOpts = ""
2044             elif detectRenames.lower() == "true":
2045                 self.diffOpts = "-M"
2046             else:
2047                 self.diffOpts = "-M%s" % detectRenames
2048
2049         # no command-line arg for -C or --find-copies-harder, just
2050         # config variables
2051         detectCopies = gitConfig("git-p4.detectCopies")
2052         if detectCopies.lower() == "false" or detectCopies == "":
2053             pass
2054         elif detectCopies.lower() == "true":
2055             self.diffOpts += " -C"
2056         else:
2057             self.diffOpts += " -C%s" % detectCopies
2058
2059         if gitConfigBool("git-p4.detectCopiesHarder"):
2060             self.diffOpts += " --find-copies-harder"
2061
2062         #
2063         # Apply the commits, one at a time.  On failure, ask if should
2064         # continue to try the rest of the patches, or quit.
2065         #
2066         if self.dry_run:
2067             print "Would apply"
2068         applied = []
2069         last = len(commits) - 1
2070         for i, commit in enumerate(commits):
2071             if self.dry_run:
2072                 print " ", read_pipe(["git", "show", "-s",
2073                                       "--format=format:%h %s", commit])
2074                 ok = True
2075             else:
2076                 ok = self.applyCommit(commit)
2077             if ok:
2078                 applied.append(commit)
2079             else:
2080                 if self.prepare_p4_only and i < last:
2081                     print "Processing only the first commit due to option" \
2082                           " --prepare-p4-only"
2083                     break
2084                 if i < last:
2085                     quit = False
2086                     while True:
2087                         # prompt for what to do, or use the option/variable
2088                         if self.conflict_behavior == "ask":
2089                             print "What do you want to do?"
2090                             response = raw_input("[s]kip this commit but apply"
2091                                                  " the rest, or [q]uit? ")
2092                             if not response:
2093                                 continue
2094                         elif self.conflict_behavior == "skip":
2095                             response = "s"
2096                         elif self.conflict_behavior == "quit":
2097                             response = "q"
2098                         else:
2099                             die("Unknown conflict_behavior '%s'" %
2100                                 self.conflict_behavior)
2101
2102                         if response[0] == "s":
2103                             print "Skipping this commit, but applying the rest"
2104                             break
2105                         if response[0] == "q":
2106                             print "Quitting"
2107                             quit = True
2108                             break
2109                     if quit:
2110                         break
2111
2112         chdir(self.oldWorkingDirectory)
2113         shelved_applied = "shelved" if self.shelve else "applied"
2114         if self.dry_run:
2115             pass
2116         elif self.prepare_p4_only:
2117             pass
2118         elif len(commits) == len(applied):
2119             print ("All commits {0}!".format(shelved_applied))
2120
2121             sync = P4Sync()
2122             if self.branch:
2123                 sync.branch = self.branch
2124             sync.run([])
2125
2126             rebase = P4Rebase()
2127             rebase.rebase()
2128
2129         else:
2130             if len(applied) == 0:
2131                 print ("No commits {0}.".format(shelved_applied))
2132             else:
2133                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2134                 for c in commits:
2135                     if c in applied:
2136                         star = "*"
2137                     else:
2138                         star = " "
2139                     print star, read_pipe(["git", "show", "-s",
2140                                            "--format=format:%h %s",  c])
2141                 print "You will have to do 'git p4 sync' and rebase."
2142
2143         if gitConfigBool("git-p4.exportLabels"):
2144             self.exportLabels = True
2145
2146         if self.exportLabels:
2147             p4Labels = getP4Labels(self.depotPath)
2148             gitTags = getGitTags()
2149
2150             missingGitTags = gitTags - p4Labels
2151             self.exportGitTags(missingGitTags)
2152
2153         # exit with error unless everything applied perfectly
2154         if len(commits) != len(applied):
2155                 sys.exit(1)
2156
2157         return True
2158
2159 class View(object):
2160     """Represent a p4 view ("p4 help views"), and map files in a
2161        repo according to the view."""
2162
2163     def __init__(self, client_name):
2164         self.mappings = []
2165         self.client_prefix = "//%s/" % client_name
2166         # cache results of "p4 where" to lookup client file locations
2167         self.client_spec_path_cache = {}
2168
2169     def append(self, view_line):
2170         """Parse a view line, splitting it into depot and client
2171            sides.  Append to self.mappings, preserving order.  This
2172            is only needed for tag creation."""
2173
2174         # Split the view line into exactly two words.  P4 enforces
2175         # structure on these lines that simplifies this quite a bit.
2176         #
2177         # Either or both words may be double-quoted.
2178         # Single quotes do not matter.
2179         # Double-quote marks cannot occur inside the words.
2180         # A + or - prefix is also inside the quotes.
2181         # There are no quotes unless they contain a space.
2182         # The line is already white-space stripped.
2183         # The two words are separated by a single space.
2184         #
2185         if view_line[0] == '"':
2186             # First word is double quoted.  Find its end.
2187             close_quote_index = view_line.find('"', 1)
2188             if close_quote_index <= 0:
2189                 die("No first-word closing quote found: %s" % view_line)
2190             depot_side = view_line[1:close_quote_index]
2191             # skip closing quote and space
2192             rhs_index = close_quote_index + 1 + 1
2193         else:
2194             space_index = view_line.find(" ")
2195             if space_index <= 0:
2196                 die("No word-splitting space found: %s" % view_line)
2197             depot_side = view_line[0:space_index]
2198             rhs_index = space_index + 1
2199
2200         # prefix + means overlay on previous mapping
2201         if depot_side.startswith("+"):
2202             depot_side = depot_side[1:]
2203
2204         # prefix - means exclude this path, leave out of mappings
2205         exclude = False
2206         if depot_side.startswith("-"):
2207             exclude = True
2208             depot_side = depot_side[1:]
2209
2210         if not exclude:
2211             self.mappings.append(depot_side)
2212
2213     def convert_client_path(self, clientFile):
2214         # chop off //client/ part to make it relative
2215         if not clientFile.startswith(self.client_prefix):
2216             die("No prefix '%s' on clientFile '%s'" %
2217                 (self.client_prefix, clientFile))
2218         return clientFile[len(self.client_prefix):]
2219
2220     def update_client_spec_path_cache(self, files):
2221         """ Caching file paths by "p4 where" batch query """
2222
2223         # List depot file paths exclude that already cached
2224         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2225
2226         if len(fileArgs) == 0:
2227             return  # All files in cache
2228
2229         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2230         for res in where_result:
2231             if "code" in res and res["code"] == "error":
2232                 # assume error is "... file(s) not in client view"
2233                 continue
2234             if "clientFile" not in res:
2235                 die("No clientFile in 'p4 where' output")
2236             if "unmap" in res:
2237                 # it will list all of them, but only one not unmap-ped
2238                 continue
2239             if gitConfigBool("core.ignorecase"):
2240                 res['depotFile'] = res['depotFile'].lower()
2241             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2242
2243         # not found files or unmap files set to ""
2244         for depotFile in fileArgs:
2245             if gitConfigBool("core.ignorecase"):
2246                 depotFile = depotFile.lower()
2247             if depotFile not in self.client_spec_path_cache:
2248                 self.client_spec_path_cache[depotFile] = ""
2249
2250     def map_in_client(self, depot_path):
2251         """Return the relative location in the client where this
2252            depot file should live.  Returns "" if the file should
2253            not be mapped in the client."""
2254
2255         if gitConfigBool("core.ignorecase"):
2256             depot_path = depot_path.lower()
2257
2258         if depot_path in self.client_spec_path_cache:
2259             return self.client_spec_path_cache[depot_path]
2260
2261         die( "Error: %s is not found in client spec path" % depot_path )
2262         return ""
2263
2264 class P4Sync(Command, P4UserMap):
2265     delete_actions = ( "delete", "move/delete", "purge" )
2266
2267     def __init__(self):
2268         Command.__init__(self)
2269         P4UserMap.__init__(self)
2270         self.options = [
2271                 optparse.make_option("--branch", dest="branch"),
2272                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2273                 optparse.make_option("--changesfile", dest="changesFile"),
2274                 optparse.make_option("--silent", dest="silent", action="store_true"),
2275                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2276                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2277                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2278                                      help="Import into refs/heads/ , not refs/remotes"),
2279                 optparse.make_option("--max-changes", dest="maxChanges",
2280                                      help="Maximum number of changes to import"),
2281                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2282                                      help="Internal block size to use when iteratively calling p4 changes"),
2283                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2284                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2285                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2286                                      help="Only sync files that are included in the Perforce Client Spec"),
2287                 optparse.make_option("-/", dest="cloneExclude",
2288                                      action="append", type="string",
2289                                      help="exclude depot path"),
2290         ]
2291         self.description = """Imports from Perforce into a git repository.\n
2292     example:
2293     //depot/my/project/ -- to import the current head
2294     //depot/my/project/@all -- to import everything
2295     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2296
2297     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2298
2299         self.usage += " //depot/path[@revRange]"
2300         self.silent = False
2301         self.createdBranches = set()
2302         self.committedChanges = set()
2303         self.branch = ""
2304         self.detectBranches = False
2305         self.detectLabels = False
2306         self.importLabels = False
2307         self.changesFile = ""
2308         self.syncWithOrigin = True
2309         self.importIntoRemotes = True
2310         self.maxChanges = ""
2311         self.changes_block_size = None
2312         self.keepRepoPath = False
2313         self.depotPaths = None
2314         self.p4BranchesInGit = []
2315         self.cloneExclude = []
2316         self.useClientSpec = False
2317         self.useClientSpec_from_options = False
2318         self.clientSpecDirs = None
2319         self.tempBranches = []
2320         self.tempBranchLocation = "refs/git-p4-tmp"
2321         self.largeFileSystem = None
2322
2323         if gitConfig('git-p4.largeFileSystem'):
2324             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2325             self.largeFileSystem = largeFileSystemConstructor(
2326                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2327             )
2328
2329         if gitConfig("git-p4.syncFromOrigin") == "false":
2330             self.syncWithOrigin = False
2331
2332     # This is required for the "append" cloneExclude action
2333     def ensure_value(self, attr, value):
2334         if not hasattr(self, attr) or getattr(self, attr) is None:
2335             setattr(self, attr, value)
2336         return getattr(self, attr)
2337
2338     # Force a checkpoint in fast-import and wait for it to finish
2339     def checkpoint(self):
2340         self.gitStream.write("checkpoint\n\n")
2341         self.gitStream.write("progress checkpoint\n\n")
2342         out = self.gitOutput.readline()
2343         if self.verbose:
2344             print "checkpoint finished: " + out
2345
2346     def extractFilesFromCommit(self, commit):
2347         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2348                              for path in self.cloneExclude]
2349         files = []
2350         fnum = 0
2351         while commit.has_key("depotFile%s" % fnum):
2352             path =  commit["depotFile%s" % fnum]
2353
2354             if [p for p in self.cloneExclude
2355                 if p4PathStartsWith(path, p)]:
2356                 found = False
2357             else:
2358                 found = [p for p in self.depotPaths
2359                          if p4PathStartsWith(path, p)]
2360             if not found:
2361                 fnum = fnum + 1
2362                 continue
2363
2364             file = {}
2365             file["path"] = path
2366             file["rev"] = commit["rev%s" % fnum]
2367             file["action"] = commit["action%s" % fnum]
2368             file["type"] = commit["type%s" % fnum]
2369             files.append(file)
2370             fnum = fnum + 1
2371         return files
2372
2373     def extractJobsFromCommit(self, commit):
2374         jobs = []
2375         jnum = 0
2376         while commit.has_key("job%s" % jnum):
2377             job = commit["job%s" % jnum]
2378             jobs.append(job)
2379             jnum = jnum + 1
2380         return jobs
2381
2382     def stripRepoPath(self, path, prefixes):
2383         """When streaming files, this is called to map a p4 depot path
2384            to where it should go in git.  The prefixes are either
2385            self.depotPaths, or self.branchPrefixes in the case of
2386            branch detection."""
2387
2388         if self.useClientSpec:
2389             # branch detection moves files up a level (the branch name)
2390             # from what client spec interpretation gives
2391             path = self.clientSpecDirs.map_in_client(path)
2392             if self.detectBranches:
2393                 for b in self.knownBranches:
2394                     if path.startswith(b + "/"):
2395                         path = path[len(b)+1:]
2396
2397         elif self.keepRepoPath:
2398             # Preserve everything in relative path name except leading
2399             # //depot/; just look at first prefix as they all should
2400             # be in the same depot.
2401             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2402             if p4PathStartsWith(path, depot):
2403                 path = path[len(depot):]
2404
2405         else:
2406             for p in prefixes:
2407                 if p4PathStartsWith(path, p):
2408                     path = path[len(p):]
2409                     break
2410
2411         path = wildcard_decode(path)
2412         return path
2413
2414     def splitFilesIntoBranches(self, commit):
2415         """Look at each depotFile in the commit to figure out to what
2416            branch it belongs."""
2417
2418         if self.clientSpecDirs:
2419             files = self.extractFilesFromCommit(commit)
2420             self.clientSpecDirs.update_client_spec_path_cache(files)
2421
2422         branches = {}
2423         fnum = 0
2424         while commit.has_key("depotFile%s" % fnum):
2425             path =  commit["depotFile%s" % fnum]
2426             found = [p for p in self.depotPaths
2427                      if p4PathStartsWith(path, p)]
2428             if not found:
2429                 fnum = fnum + 1
2430                 continue
2431
2432             file = {}
2433             file["path"] = path
2434             file["rev"] = commit["rev%s" % fnum]
2435             file["action"] = commit["action%s" % fnum]
2436             file["type"] = commit["type%s" % fnum]
2437             fnum = fnum + 1
2438
2439             # start with the full relative path where this file would
2440             # go in a p4 client
2441             if self.useClientSpec:
2442                 relPath = self.clientSpecDirs.map_in_client(path)
2443             else:
2444                 relPath = self.stripRepoPath(path, self.depotPaths)
2445
2446             for branch in self.knownBranches.keys():
2447                 # add a trailing slash so that a commit into qt/4.2foo
2448                 # doesn't end up in qt/4.2, e.g.
2449                 if relPath.startswith(branch + "/"):
2450                     if branch not in branches:
2451                         branches[branch] = []
2452                     branches[branch].append(file)
2453                     break
2454
2455         return branches
2456
2457     def writeToGitStream(self, gitMode, relPath, contents):
2458         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2459         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2460         for d in contents:
2461             self.gitStream.write(d)
2462         self.gitStream.write('\n')
2463
2464     # output one file from the P4 stream
2465     # - helper for streamP4Files
2466
2467     def streamOneP4File(self, file, contents):
2468         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2469         if verbose:
2470             size = int(self.stream_file['fileSize'])
2471             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2472             sys.stdout.flush()
2473
2474         (type_base, type_mods) = split_p4_type(file["type"])
2475
2476         git_mode = "100644"
2477         if "x" in type_mods:
2478             git_mode = "100755"
2479         if type_base == "symlink":
2480             git_mode = "120000"
2481             # p4 print on a symlink sometimes contains "target\n";
2482             # if it does, remove the newline
2483             data = ''.join(contents)
2484             if not data:
2485                 # Some version of p4 allowed creating a symlink that pointed
2486                 # to nothing.  This causes p4 errors when checking out such
2487                 # a change, and errors here too.  Work around it by ignoring
2488                 # the bad symlink; hopefully a future change fixes it.
2489                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2490                 return
2491             elif data[-1] == '\n':
2492                 contents = [data[:-1]]
2493             else:
2494                 contents = [data]
2495
2496         if type_base == "utf16":
2497             # p4 delivers different text in the python output to -G
2498             # than it does when using "print -o", or normal p4 client
2499             # operations.  utf16 is converted to ascii or utf8, perhaps.
2500             # But ascii text saved as -t utf16 is completely mangled.
2501             # Invoke print -o to get the real contents.
2502             #
2503             # On windows, the newlines will always be mangled by print, so put
2504             # them back too.  This is not needed to the cygwin windows version,
2505             # just the native "NT" type.
2506             #
2507             try:
2508                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2509             except Exception as e:
2510                 if 'Translation of file content failed' in str(e):
2511                     type_base = 'binary'
2512                 else:
2513                     raise e
2514             else:
2515                 if p4_version_string().find('/NT') >= 0:
2516                     text = text.replace('\r\n', '\n')
2517                 contents = [ text ]
2518
2519         if type_base == "apple":
2520             # Apple filetype files will be streamed as a concatenation of
2521             # its appledouble header and the contents.  This is useless
2522             # on both macs and non-macs.  If using "print -q -o xx", it
2523             # will create "xx" with the data, and "%xx" with the header.
2524             # This is also not very useful.
2525             #
2526             # Ideally, someday, this script can learn how to generate
2527             # appledouble files directly and import those to git, but
2528             # non-mac machines can never find a use for apple filetype.
2529             print "\nIgnoring apple filetype file %s" % file['depotFile']
2530             return
2531
2532         # Note that we do not try to de-mangle keywords on utf16 files,
2533         # even though in theory somebody may want that.
2534         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2535         if pattern:
2536             regexp = re.compile(pattern, re.VERBOSE)
2537             text = ''.join(contents)
2538             text = regexp.sub(r'$\1$', text)
2539             contents = [ text ]
2540
2541         try:
2542             relPath.decode('ascii')
2543         except:
2544             encoding = 'utf8'
2545             if gitConfig('git-p4.pathEncoding'):
2546                 encoding = gitConfig('git-p4.pathEncoding')
2547             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2548             if self.verbose:
2549                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2550
2551         if self.largeFileSystem:
2552             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2553
2554         self.writeToGitStream(git_mode, relPath, contents)
2555
2556     def streamOneP4Deletion(self, file):
2557         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2558         if verbose:
2559             sys.stdout.write("delete %s\n" % relPath)
2560             sys.stdout.flush()
2561         self.gitStream.write("D %s\n" % relPath)
2562
2563         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2564             self.largeFileSystem.removeLargeFile(relPath)
2565
2566     # handle another chunk of streaming data
2567     def streamP4FilesCb(self, marshalled):
2568
2569         # catch p4 errors and complain
2570         err = None
2571         if "code" in marshalled:
2572             if marshalled["code"] == "error":
2573                 if "data" in marshalled:
2574                     err = marshalled["data"].rstrip()
2575
2576         if not err and 'fileSize' in self.stream_file:
2577             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2578             if required_bytes > 0:
2579                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2580                     os.getcwd(), required_bytes/1024/1024
2581                 )
2582
2583         if err:
2584             f = None
2585             if self.stream_have_file_info:
2586                 if "depotFile" in self.stream_file:
2587                     f = self.stream_file["depotFile"]
2588             # force a failure in fast-import, else an empty
2589             # commit will be made
2590             self.gitStream.write("\n")
2591             self.gitStream.write("die-now\n")
2592             self.gitStream.close()
2593             # ignore errors, but make sure it exits first
2594             self.importProcess.wait()
2595             if f:
2596                 die("Error from p4 print for %s: %s" % (f, err))
2597             else:
2598                 die("Error from p4 print: %s" % err)
2599
2600         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2601             # start of a new file - output the old one first
2602             self.streamOneP4File(self.stream_file, self.stream_contents)
2603             self.stream_file = {}
2604             self.stream_contents = []
2605             self.stream_have_file_info = False
2606
2607         # pick up the new file information... for the
2608         # 'data' field we need to append to our array
2609         for k in marshalled.keys():
2610             if k == 'data':
2611                 if 'streamContentSize' not in self.stream_file:
2612                     self.stream_file['streamContentSize'] = 0
2613                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2614                 self.stream_contents.append(marshalled['data'])
2615             else:
2616                 self.stream_file[k] = marshalled[k]
2617
2618         if (verbose and
2619             'streamContentSize' in self.stream_file and
2620             'fileSize' in self.stream_file and
2621             'depotFile' in self.stream_file):
2622             size = int(self.stream_file["fileSize"])
2623             if size > 0:
2624                 progress = 100*self.stream_file['streamContentSize']/size
2625                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2626                 sys.stdout.flush()
2627
2628         self.stream_have_file_info = True
2629
2630     # Stream directly from "p4 files" into "git fast-import"
2631     def streamP4Files(self, files):
2632         filesForCommit = []
2633         filesToRead = []
2634         filesToDelete = []
2635
2636         for f in files:
2637             filesForCommit.append(f)
2638             if f['action'] in self.delete_actions:
2639                 filesToDelete.append(f)
2640             else:
2641                 filesToRead.append(f)
2642
2643         # deleted files...
2644         for f in filesToDelete:
2645             self.streamOneP4Deletion(f)
2646
2647         if len(filesToRead) > 0:
2648             self.stream_file = {}
2649             self.stream_contents = []
2650             self.stream_have_file_info = False
2651
2652             # curry self argument
2653             def streamP4FilesCbSelf(entry):
2654                 self.streamP4FilesCb(entry)
2655
2656             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2657
2658             p4CmdList(["-x", "-", "print"],
2659                       stdin=fileArgs,
2660                       cb=streamP4FilesCbSelf)
2661
2662             # do the last chunk
2663             if self.stream_file.has_key('depotFile'):
2664                 self.streamOneP4File(self.stream_file, self.stream_contents)
2665
2666     def make_email(self, userid):
2667         if userid in self.users:
2668             return self.users[userid]
2669         else:
2670             return "%s <a@b>" % userid
2671
2672     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2673         """ Stream a p4 tag.
2674         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2675         """
2676
2677         if verbose:
2678             print "writing tag %s for commit %s" % (labelName, commit)
2679         gitStream.write("tag %s\n" % labelName)
2680         gitStream.write("from %s\n" % commit)
2681
2682         if labelDetails.has_key('Owner'):
2683             owner = labelDetails["Owner"]
2684         else:
2685             owner = None
2686
2687         # Try to use the owner of the p4 label, or failing that,
2688         # the current p4 user id.
2689         if owner:
2690             email = self.make_email(owner)
2691         else:
2692             email = self.make_email(self.p4UserId())
2693         tagger = "%s %s %s" % (email, epoch, self.tz)
2694
2695         gitStream.write("tagger %s\n" % tagger)
2696
2697         print "labelDetails=",labelDetails
2698         if labelDetails.has_key('Description'):
2699             description = labelDetails['Description']
2700         else:
2701             description = 'Label from git p4'
2702
2703         gitStream.write("data %d\n" % len(description))
2704         gitStream.write(description)
2705         gitStream.write("\n")
2706
2707     def inClientSpec(self, path):
2708         if not self.clientSpecDirs:
2709             return True
2710         inClientSpec = self.clientSpecDirs.map_in_client(path)
2711         if not inClientSpec and self.verbose:
2712             print('Ignoring file outside of client spec: {0}'.format(path))
2713         return inClientSpec
2714
2715     def hasBranchPrefix(self, path):
2716         if not self.branchPrefixes:
2717             return True
2718         hasPrefix = [p for p in self.branchPrefixes
2719                         if p4PathStartsWith(path, p)]
2720         if not hasPrefix and self.verbose:
2721             print('Ignoring file outside of prefix: {0}'.format(path))
2722         return hasPrefix
2723
2724     def commit(self, details, files, branch, parent = ""):
2725         epoch = details["time"]
2726         author = details["user"]
2727         jobs = self.extractJobsFromCommit(details)
2728
2729         if self.verbose:
2730             print('commit into {0}'.format(branch))
2731
2732         if self.clientSpecDirs:
2733             self.clientSpecDirs.update_client_spec_path_cache(files)
2734
2735         files = [f for f in files
2736             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2737
2738         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2739             print('Ignoring revision {0} as it would produce an empty commit.'
2740                 .format(details['change']))
2741             return
2742
2743         self.gitStream.write("commit %s\n" % branch)
2744         self.gitStream.write("mark :%s\n" % details["change"])
2745         self.committedChanges.add(int(details["change"]))
2746         committer = ""
2747         if author not in self.users:
2748             self.getUserMapFromPerforceServer()
2749         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2750
2751         self.gitStream.write("committer %s\n" % committer)
2752
2753         self.gitStream.write("data <<EOT\n")
2754         self.gitStream.write(details["desc"])
2755         if len(jobs) > 0:
2756             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2757         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2758                              (','.join(self.branchPrefixes), details["change"]))
2759         if len(details['options']) > 0:
2760             self.gitStream.write(": options = %s" % details['options'])
2761         self.gitStream.write("]\nEOT\n\n")
2762
2763         if len(parent) > 0:
2764             if self.verbose:
2765                 print "parent %s" % parent
2766             self.gitStream.write("from %s\n" % parent)
2767
2768         self.streamP4Files(files)
2769         self.gitStream.write("\n")
2770
2771         change = int(details["change"])
2772
2773         if self.labels.has_key(change):
2774             label = self.labels[change]
2775             labelDetails = label[0]
2776             labelRevisions = label[1]
2777             if self.verbose:
2778                 print "Change %s is labelled %s" % (change, labelDetails)
2779
2780             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2781                                                 for p in self.branchPrefixes])
2782
2783             if len(files) == len(labelRevisions):
2784
2785                 cleanedFiles = {}
2786                 for info in files:
2787                     if info["action"] in self.delete_actions:
2788                         continue
2789                     cleanedFiles[info["depotFile"]] = info["rev"]
2790
2791                 if cleanedFiles == labelRevisions:
2792                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2793
2794                 else:
2795                     if not self.silent:
2796                         print ("Tag %s does not match with change %s: files do not match."
2797                                % (labelDetails["label"], change))
2798
2799             else:
2800                 if not self.silent:
2801                     print ("Tag %s does not match with change %s: file count is different."
2802                            % (labelDetails["label"], change))
2803
2804     # Build a dictionary of changelists and labels, for "detect-labels" option.
2805     def getLabels(self):
2806         self.labels = {}
2807
2808         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2809         if len(l) > 0 and not self.silent:
2810             print "Finding files belonging to labels in %s" % `self.depotPaths`
2811
2812         for output in l:
2813             label = output["label"]
2814             revisions = {}
2815             newestChange = 0
2816             if self.verbose:
2817                 print "Querying files for label %s" % label
2818             for file in p4CmdList(["files"] +
2819                                       ["%s...@%s" % (p, label)
2820                                           for p in self.depotPaths]):
2821                 revisions[file["depotFile"]] = file["rev"]
2822                 change = int(file["change"])
2823                 if change > newestChange:
2824                     newestChange = change
2825
2826             self.labels[newestChange] = [output, revisions]
2827
2828         if self.verbose:
2829             print "Label changes: %s" % self.labels.keys()
2830
2831     # Import p4 labels as git tags. A direct mapping does not
2832     # exist, so assume that if all the files are at the same revision
2833     # then we can use that, or it's something more complicated we should
2834     # just ignore.
2835     def importP4Labels(self, stream, p4Labels):
2836         if verbose:
2837             print "import p4 labels: " + ' '.join(p4Labels)
2838
2839         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2840         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2841         if len(validLabelRegexp) == 0:
2842             validLabelRegexp = defaultLabelRegexp
2843         m = re.compile(validLabelRegexp)
2844
2845         for name in p4Labels:
2846             commitFound = False
2847
2848             if not m.match(name):
2849                 if verbose:
2850                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2851                 continue
2852
2853             if name in ignoredP4Labels:
2854                 continue
2855
2856             labelDetails = p4CmdList(['label', "-o", name])[0]
2857
2858             # get the most recent changelist for each file in this label
2859             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2860                                 for p in self.depotPaths])
2861
2862             if change.has_key('change'):
2863                 # find the corresponding git commit; take the oldest commit
2864                 changelist = int(change['change'])
2865                 if changelist in self.committedChanges:
2866                     gitCommit = ":%d" % changelist       # use a fast-import mark
2867                     commitFound = True
2868                 else:
2869                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2870                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2871                     if len(gitCommit) == 0:
2872                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2873                     else:
2874                         commitFound = True
2875                         gitCommit = gitCommit.strip()
2876
2877                 if commitFound:
2878                     # Convert from p4 time format
2879                     try:
2880                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2881                     except ValueError:
2882                         print "Could not convert label time %s" % labelDetails['Update']
2883                         tmwhen = 1
2884
2885                     when = int(time.mktime(tmwhen))
2886                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2887                     if verbose:
2888                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2889             else:
2890                 if verbose:
2891                     print "Label %s has no changelists - possibly deleted?" % name
2892
2893             if not commitFound:
2894                 # We can't import this label; don't try again as it will get very
2895                 # expensive repeatedly fetching all the files for labels that will
2896                 # never be imported. If the label is moved in the future, the
2897                 # ignore will need to be removed manually.
2898                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2899
2900     def guessProjectName(self):
2901         for p in self.depotPaths:
2902             if p.endswith("/"):
2903                 p = p[:-1]
2904             p = p[p.strip().rfind("/") + 1:]
2905             if not p.endswith("/"):
2906                p += "/"
2907             return p
2908
2909     def getBranchMapping(self):
2910         lostAndFoundBranches = set()
2911
2912         user = gitConfig("git-p4.branchUser")
2913         if len(user) > 0:
2914             command = "branches -u %s" % user
2915         else:
2916             command = "branches"
2917
2918         for info in p4CmdList(command):
2919             details = p4Cmd(["branch", "-o", info["branch"]])
2920             viewIdx = 0
2921             while details.has_key("View%s" % viewIdx):
2922                 paths = details["View%s" % viewIdx].split(" ")
2923                 viewIdx = viewIdx + 1
2924                 # require standard //depot/foo/... //depot/bar/... mapping
2925                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2926                     continue
2927                 source = paths[0]
2928                 destination = paths[1]
2929                 ## HACK
2930                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2931                     source = source[len(self.depotPaths[0]):-4]
2932                     destination = destination[len(self.depotPaths[0]):-4]
2933
2934                     if destination in self.knownBranches:
2935                         if not self.silent:
2936                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2937                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2938                         continue
2939
2940                     self.knownBranches[destination] = source
2941
2942                     lostAndFoundBranches.discard(destination)
2943
2944                     if source not in self.knownBranches:
2945                         lostAndFoundBranches.add(source)
2946
2947         # Perforce does not strictly require branches to be defined, so we also
2948         # check git config for a branch list.
2949         #
2950         # Example of branch definition in git config file:
2951         # [git-p4]
2952         #   branchList=main:branchA
2953         #   branchList=main:branchB
2954         #   branchList=branchA:branchC
2955         configBranches = gitConfigList("git-p4.branchList")
2956         for branch in configBranches:
2957             if branch:
2958                 (source, destination) = branch.split(":")
2959                 self.knownBranches[destination] = source
2960
2961                 lostAndFoundBranches.discard(destination)
2962
2963                 if source not in self.knownBranches:
2964                     lostAndFoundBranches.add(source)
2965
2966
2967         for branch in lostAndFoundBranches:
2968             self.knownBranches[branch] = branch
2969
2970     def getBranchMappingFromGitBranches(self):
2971         branches = p4BranchesInGit(self.importIntoRemotes)
2972         for branch in branches.keys():
2973             if branch == "master":
2974                 branch = "main"
2975             else:
2976                 branch = branch[len(self.projectName):]
2977             self.knownBranches[branch] = branch
2978
2979     def updateOptionDict(self, d):
2980         option_keys = {}
2981         if self.keepRepoPath:
2982             option_keys['keepRepoPath'] = 1
2983
2984         d["options"] = ' '.join(sorted(option_keys.keys()))
2985
2986     def readOptions(self, d):
2987         self.keepRepoPath = (d.has_key('options')
2988                              and ('keepRepoPath' in d['options']))
2989
2990     def gitRefForBranch(self, branch):
2991         if branch == "main":
2992             return self.refPrefix + "master"
2993
2994         if len(branch) <= 0:
2995             return branch
2996
2997         return self.refPrefix + self.projectName + branch
2998
2999     def gitCommitByP4Change(self, ref, change):
3000         if self.verbose:
3001             print "looking in ref " + ref + " for change %s using bisect..." % change
3002
3003         earliestCommit = ""
3004         latestCommit = parseRevision(ref)
3005
3006         while True:
3007             if self.verbose:
3008                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3009             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3010             if len(next) == 0:
3011                 if self.verbose:
3012                     print "argh"
3013                 return ""
3014             log = extractLogMessageFromGitCommit(next)
3015             settings = extractSettingsGitLog(log)
3016             currentChange = int(settings['change'])
3017             if self.verbose:
3018                 print "current change %s" % currentChange
3019
3020             if currentChange == change:
3021                 if self.verbose:
3022                     print "found %s" % next
3023                 return next
3024
3025             if currentChange < change:
3026                 earliestCommit = "^%s" % next
3027             else:
3028                 latestCommit = "%s" % next
3029
3030         return ""
3031
3032     def importNewBranch(self, branch, maxChange):
3033         # make fast-import flush all changes to disk and update the refs using the checkpoint
3034         # command so that we can try to find the branch parent in the git history
3035         self.gitStream.write("checkpoint\n\n");
3036         self.gitStream.flush();
3037         branchPrefix = self.depotPaths[0] + branch + "/"
3038         range = "@1,%s" % maxChange
3039         #print "prefix" + branchPrefix
3040         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3041         if len(changes) <= 0:
3042             return False
3043         firstChange = changes[0]
3044         #print "first change in branch: %s" % firstChange
3045         sourceBranch = self.knownBranches[branch]
3046         sourceDepotPath = self.depotPaths[0] + sourceBranch
3047         sourceRef = self.gitRefForBranch(sourceBranch)
3048         #print "source " + sourceBranch
3049
3050         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3051         #print "branch parent: %s" % branchParentChange
3052         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3053         if len(gitParent) > 0:
3054             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3055             #print "parent git commit: %s" % gitParent
3056
3057         self.importChanges(changes)
3058         return True
3059
3060     def searchParent(self, parent, branch, target):
3061         parentFound = False
3062         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3063                                      "--no-merges", parent]):
3064             blob = blob.strip()
3065             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3066                 parentFound = True
3067                 if self.verbose:
3068                     print "Found parent of %s in commit %s" % (branch, blob)
3069                 break
3070         if parentFound:
3071             return blob
3072         else:
3073             return None
3074
3075     def importChanges(self, changes):
3076         cnt = 1
3077         for change in changes:
3078             description = p4_describe(change)
3079             self.updateOptionDict(description)
3080
3081             if not self.silent:
3082                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3083                 sys.stdout.flush()
3084             cnt = cnt + 1
3085
3086             try:
3087                 if self.detectBranches:
3088                     branches = self.splitFilesIntoBranches(description)
3089                     for branch in branches.keys():
3090                         ## HACK  --hwn
3091                         branchPrefix = self.depotPaths[0] + branch + "/"
3092                         self.branchPrefixes = [ branchPrefix ]
3093
3094                         parent = ""
3095
3096                         filesForCommit = branches[branch]
3097
3098                         if self.verbose:
3099                             print "branch is %s" % branch
3100
3101                         self.updatedBranches.add(branch)
3102
3103                         if branch not in self.createdBranches:
3104                             self.createdBranches.add(branch)
3105                             parent = self.knownBranches[branch]
3106                             if parent == branch:
3107                                 parent = ""
3108                             else:
3109                                 fullBranch = self.projectName + branch
3110                                 if fullBranch not in self.p4BranchesInGit:
3111                                     if not self.silent:
3112                                         print("\n    Importing new branch %s" % fullBranch);
3113                                     if self.importNewBranch(branch, change - 1):
3114                                         parent = ""
3115                                         self.p4BranchesInGit.append(fullBranch)
3116                                     if not self.silent:
3117                                         print("\n    Resuming with change %s" % change);
3118
3119                                 if self.verbose:
3120                                     print "parent determined through known branches: %s" % parent
3121
3122                         branch = self.gitRefForBranch(branch)
3123                         parent = self.gitRefForBranch(parent)
3124
3125                         if self.verbose:
3126                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3127
3128                         if len(parent) == 0 and branch in self.initialParents:
3129                             parent = self.initialParents[branch]
3130                             del self.initialParents[branch]
3131
3132                         blob = None
3133                         if len(parent) > 0:
3134                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3135                             if self.verbose:
3136                                 print "Creating temporary branch: " + tempBranch
3137                             self.commit(description, filesForCommit, tempBranch)
3138                             self.tempBranches.append(tempBranch)
3139                             self.checkpoint()
3140                             blob = self.searchParent(parent, branch, tempBranch)
3141                         if blob:
3142                             self.commit(description, filesForCommit, branch, blob)
3143                         else:
3144                             if self.verbose:
3145                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3146                             self.commit(description, filesForCommit, branch, parent)
3147                 else:
3148                     files = self.extractFilesFromCommit(description)
3149                     self.commit(description, files, self.branch,
3150                                 self.initialParent)
3151                     # only needed once, to connect to the previous commit
3152                     self.initialParent = ""
3153             except IOError:
3154                 print self.gitError.read()
3155                 sys.exit(1)
3156
3157     def importHeadRevision(self, revision):
3158         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3159
3160         details = {}
3161         details["user"] = "git perforce import user"
3162         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3163                            % (' '.join(self.depotPaths), revision))
3164         details["change"] = revision
3165         newestRevision = 0
3166
3167         fileCnt = 0
3168         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3169
3170         for info in p4CmdList(["files"] + fileArgs):
3171
3172             if 'code' in info and info['code'] == 'error':
3173                 sys.stderr.write("p4 returned an error: %s\n"
3174                                  % info['data'])
3175                 if info['data'].find("must refer to client") >= 0:
3176                     sys.stderr.write("This particular p4 error is misleading.\n")
3177                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3178                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3179                 sys.exit(1)
3180             if 'p4ExitCode' in info:
3181                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3182                 sys.exit(1)
3183
3184
3185             change = int(info["change"])
3186             if change > newestRevision:
3187                 newestRevision = change
3188
3189             if info["action"] in self.delete_actions:
3190                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3191                 #fileCnt = fileCnt + 1
3192                 continue
3193
3194             for prop in ["depotFile", "rev", "action", "type" ]:
3195                 details["%s%s" % (prop, fileCnt)] = info[prop]
3196
3197             fileCnt = fileCnt + 1
3198
3199         details["change"] = newestRevision
3200
3201         # Use time from top-most change so that all git p4 clones of
3202         # the same p4 repo have the same commit SHA1s.
3203         res = p4_describe(newestRevision)
3204         details["time"] = res["time"]
3205
3206         self.updateOptionDict(details)
3207         try:
3208             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3209         except IOError:
3210             print "IO error with git fast-import. Is your git version recent enough?"
3211             print self.gitError.read()
3212
3213
3214     def run(self, args):
3215         self.depotPaths = []
3216         self.changeRange = ""
3217         self.previousDepotPaths = []
3218         self.hasOrigin = False
3219
3220         # map from branch depot path to parent branch
3221         self.knownBranches = {}
3222         self.initialParents = {}
3223
3224         if self.importIntoRemotes:
3225             self.refPrefix = "refs/remotes/p4/"
3226         else:
3227             self.refPrefix = "refs/heads/p4/"
3228
3229         if self.syncWithOrigin:
3230             self.hasOrigin = originP4BranchesExist()
3231             if self.hasOrigin:
3232                 if not self.silent:
3233                     print 'Syncing with origin first, using "git fetch origin"'
3234                 system("git fetch origin")
3235
3236         branch_arg_given = bool(self.branch)
3237         if len(self.branch) == 0:
3238             self.branch = self.refPrefix + "master"
3239             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3240                 system("git update-ref %s refs/heads/p4" % self.branch)
3241                 system("git branch -D p4")
3242
3243         # accept either the command-line option, or the configuration variable
3244         if self.useClientSpec:
3245             # will use this after clone to set the variable
3246             self.useClientSpec_from_options = True
3247         else:
3248             if gitConfigBool("git-p4.useclientspec"):
3249                 self.useClientSpec = True
3250         if self.useClientSpec:
3251             self.clientSpecDirs = getClientSpec()
3252
3253         # TODO: should always look at previous commits,
3254         # merge with previous imports, if possible.
3255         if args == []:
3256             if self.hasOrigin:
3257                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3258
3259             # branches holds mapping from branch name to sha1
3260             branches = p4BranchesInGit(self.importIntoRemotes)
3261
3262             # restrict to just this one, disabling detect-branches
3263             if branch_arg_given:
3264                 short = self.branch.split("/")[-1]
3265                 if short in branches:
3266                     self.p4BranchesInGit = [ short ]
3267             else:
3268                 self.p4BranchesInGit = branches.keys()
3269
3270             if len(self.p4BranchesInGit) > 1:
3271                 if not self.silent:
3272                     print "Importing from/into multiple branches"
3273                 self.detectBranches = True
3274                 for branch in branches.keys():
3275                     self.initialParents[self.refPrefix + branch] = \
3276                         branches[branch]
3277
3278             if self.verbose:
3279                 print "branches: %s" % self.p4BranchesInGit
3280
3281             p4Change = 0
3282             for branch in self.p4BranchesInGit:
3283                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3284
3285                 settings = extractSettingsGitLog(logMsg)
3286
3287                 self.readOptions(settings)
3288                 if (settings.has_key('depot-paths')
3289                     and settings.has_key ('change')):
3290                     change = int(settings['change']) + 1
3291                     p4Change = max(p4Change, change)
3292
3293                     depotPaths = sorted(settings['depot-paths'])
3294                     if self.previousDepotPaths == []:
3295                         self.previousDepotPaths = depotPaths
3296                     else:
3297                         paths = []
3298                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3299                             prev_list = prev.split("/")
3300                             cur_list = cur.split("/")
3301                             for i in range(0, min(len(cur_list), len(prev_list))):
3302                                 if cur_list[i] <> prev_list[i]:
3303                                     i = i - 1
3304                                     break
3305
3306                             paths.append ("/".join(cur_list[:i + 1]))
3307
3308                         self.previousDepotPaths = paths
3309
3310             if p4Change > 0:
3311                 self.depotPaths = sorted(self.previousDepotPaths)
3312                 self.changeRange = "@%s,#head" % p4Change
3313                 if not self.silent and not self.detectBranches:
3314                     print "Performing incremental import into %s git branch" % self.branch
3315
3316         # accept multiple ref name abbreviations:
3317         #    refs/foo/bar/branch -> use it exactly
3318         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3319         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3320         if not self.branch.startswith("refs/"):
3321             if self.importIntoRemotes:
3322                 prepend = "refs/remotes/"
3323             else:
3324                 prepend = "refs/heads/"
3325             if not self.branch.startswith("p4/"):
3326                 prepend += "p4/"
3327             self.branch = prepend + self.branch
3328
3329         if len(args) == 0 and self.depotPaths:
3330             if not self.silent:
3331                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3332         else:
3333             if self.depotPaths and self.depotPaths != args:
3334                 print ("previous import used depot path %s and now %s was specified. "
3335                        "This doesn't work!" % (' '.join (self.depotPaths),
3336                                                ' '.join (args)))
3337                 sys.exit(1)
3338
3339             self.depotPaths = sorted(args)
3340
3341         revision = ""
3342         self.users = {}
3343
3344         # Make sure no revision specifiers are used when --changesfile
3345         # is specified.
3346         bad_changesfile = False
3347         if len(self.changesFile) > 0:
3348             for p in self.depotPaths:
3349                 if p.find("@") >= 0 or p.find("#") >= 0:
3350                     bad_changesfile = True
3351                     break
3352         if bad_changesfile:
3353             die("Option --changesfile is incompatible with revision specifiers")
3354
3355         newPaths = []
3356         for p in self.depotPaths:
3357             if p.find("@") != -1:
3358                 atIdx = p.index("@")
3359                 self.changeRange = p[atIdx:]
3360                 if self.changeRange == "@all":
3361                     self.changeRange = ""
3362                 elif ',' not in self.changeRange:
3363                     revision = self.changeRange
3364                     self.changeRange = ""
3365                 p = p[:atIdx]
3366             elif p.find("#") != -1:
3367                 hashIdx = p.index("#")
3368                 revision = p[hashIdx:]
3369                 p = p[:hashIdx]
3370             elif self.previousDepotPaths == []:
3371                 # pay attention to changesfile, if given, else import
3372                 # the entire p4 tree at the head revision
3373                 if len(self.changesFile) == 0:
3374                     revision = "#head"
3375
3376             p = re.sub ("\.\.\.$", "", p)
3377             if not p.endswith("/"):
3378                 p += "/"
3379
3380             newPaths.append(p)
3381
3382         self.depotPaths = newPaths
3383
3384         # --detect-branches may change this for each branch
3385         self.branchPrefixes = self.depotPaths
3386
3387         self.loadUserMapFromCache()
3388         self.labels = {}
3389         if self.detectLabels:
3390             self.getLabels();
3391
3392         if self.detectBranches:
3393             ## FIXME - what's a P4 projectName ?
3394             self.projectName = self.guessProjectName()
3395
3396             if self.hasOrigin:
3397                 self.getBranchMappingFromGitBranches()
3398             else:
3399                 self.getBranchMapping()
3400             if self.verbose:
3401                 print "p4-git branches: %s" % self.p4BranchesInGit
3402                 print "initial parents: %s" % self.initialParents
3403             for b in self.p4BranchesInGit:
3404                 if b != "master":
3405
3406                     ## FIXME
3407                     b = b[len(self.projectName):]
3408                 self.createdBranches.add(b)
3409
3410         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3411
3412         self.importProcess = subprocess.Popen(["git", "fast-import"],
3413                                               stdin=subprocess.PIPE,
3414                                               stdout=subprocess.PIPE,
3415                                               stderr=subprocess.PIPE);
3416         self.gitOutput = self.importProcess.stdout
3417         self.gitStream = self.importProcess.stdin
3418         self.gitError = self.importProcess.stderr
3419
3420         if revision:
3421             self.importHeadRevision(revision)
3422         else:
3423             changes = []
3424
3425             if len(self.changesFile) > 0:
3426                 output = open(self.changesFile).readlines()
3427                 changeSet = set()
3428                 for line in output:
3429                     changeSet.add(int(line))
3430
3431                 for change in changeSet:
3432                     changes.append(change)
3433
3434                 changes.sort()
3435             else:
3436                 # catch "git p4 sync" with no new branches, in a repo that
3437                 # does not have any existing p4 branches
3438                 if len(args) == 0:
3439                     if not self.p4BranchesInGit:
3440                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3441
3442                     # The default branch is master, unless --branch is used to
3443                     # specify something else.  Make sure it exists, or complain
3444                     # nicely about how to use --branch.
3445                     if not self.detectBranches:
3446                         if not branch_exists(self.branch):
3447                             if branch_arg_given:
3448                                 die("Error: branch %s does not exist." % self.branch)
3449                             else:
3450                                 die("Error: no branch %s; perhaps specify one with --branch." %
3451                                     self.branch)
3452
3453                 if self.verbose:
3454                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3455                                                               self.changeRange)
3456                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3457
3458                 if len(self.maxChanges) > 0:
3459                     changes = changes[:min(int(self.maxChanges), len(changes))]
3460
3461             if len(changes) == 0:
3462                 if not self.silent:
3463                     print "No changes to import!"
3464             else:
3465                 if not self.silent and not self.detectBranches:
3466                     print "Import destination: %s" % self.branch
3467
3468                 self.updatedBranches = set()
3469
3470                 if not self.detectBranches:
3471                     if args:
3472                         # start a new branch
3473                         self.initialParent = ""
3474                     else:
3475                         # build on a previous revision
3476                         self.initialParent = parseRevision(self.branch)
3477
3478                 self.importChanges(changes)
3479
3480                 if not self.silent:
3481                     print ""
3482                     if len(self.updatedBranches) > 0:
3483                         sys.stdout.write("Updated branches: ")
3484                         for b in self.updatedBranches:
3485                             sys.stdout.write("%s " % b)
3486                         sys.stdout.write("\n")
3487
3488         if gitConfigBool("git-p4.importLabels"):
3489             self.importLabels = True
3490
3491         if self.importLabels:
3492             p4Labels = getP4Labels(self.depotPaths)
3493             gitTags = getGitTags()
3494
3495             missingP4Labels = p4Labels - gitTags
3496             self.importP4Labels(self.gitStream, missingP4Labels)
3497
3498         self.gitStream.close()
3499         if self.importProcess.wait() != 0:
3500             die("fast-import failed: %s" % self.gitError.read())
3501         self.gitOutput.close()
3502         self.gitError.close()
3503
3504         # Cleanup temporary branches created during import
3505         if self.tempBranches != []:
3506             for branch in self.tempBranches:
3507                 read_pipe("git update-ref -d %s" % branch)
3508             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3509
3510         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3511         # a convenient shortcut refname "p4".
3512         if self.importIntoRemotes:
3513             head_ref = self.refPrefix + "HEAD"
3514             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3515                 system(["git", "symbolic-ref", head_ref, self.branch])
3516
3517         return True
3518
3519 class P4Rebase(Command):
3520     def __init__(self):
3521         Command.__init__(self)
3522         self.options = [
3523                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3524         ]
3525         self.importLabels = False
3526         self.description = ("Fetches the latest revision from perforce and "
3527                             + "rebases the current work (branch) against it")
3528
3529     def run(self, args):
3530         sync = P4Sync()
3531         sync.importLabels = self.importLabels
3532         sync.run([])
3533
3534         return self.rebase()
3535
3536     def rebase(self):
3537         if os.system("git update-index --refresh") != 0:
3538             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3539         if len(read_pipe("git diff-index HEAD --")) > 0:
3540             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3541
3542         [upstream, settings] = findUpstreamBranchPoint()
3543         if len(upstream) == 0:
3544             die("Cannot find upstream branchpoint for rebase")
3545
3546         # the branchpoint may be p4/foo~3, so strip off the parent
3547         upstream = re.sub("~[0-9]+$", "", upstream)
3548
3549         print "Rebasing the current branch onto %s" % upstream
3550         oldHead = read_pipe("git rev-parse HEAD").strip()
3551         system("git rebase %s" % upstream)
3552         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3553         return True
3554
3555 class P4Clone(P4Sync):
3556     def __init__(self):
3557         P4Sync.__init__(self)
3558         self.description = "Creates a new git repository and imports from Perforce into it"
3559         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3560         self.options += [
3561             optparse.make_option("--destination", dest="cloneDestination",
3562                                  action='store', default=None,
3563                                  help="where to leave result of the clone"),
3564             optparse.make_option("--bare", dest="cloneBare",
3565                                  action="store_true", default=False),
3566         ]
3567         self.cloneDestination = None
3568         self.needsGit = False
3569         self.cloneBare = False
3570
3571     def defaultDestination(self, args):
3572         ## TODO: use common prefix of args?
3573         depotPath = args[0]
3574         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3575         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3576         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3577         depotDir = re.sub(r"/$", "", depotDir)
3578         return os.path.split(depotDir)[1]
3579
3580     def run(self, args):
3581         if len(args) < 1:
3582             return False
3583
3584         if self.keepRepoPath and not self.cloneDestination:
3585             sys.stderr.write("Must specify destination for --keep-path\n")
3586             sys.exit(1)
3587
3588         depotPaths = args
3589
3590         if not self.cloneDestination and len(depotPaths) > 1:
3591             self.cloneDestination = depotPaths[-1]
3592             depotPaths = depotPaths[:-1]
3593
3594         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3595         for p in depotPaths:
3596             if not p.startswith("//"):
3597                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3598                 return False
3599
3600         if not self.cloneDestination:
3601             self.cloneDestination = self.defaultDestination(args)
3602
3603         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3604
3605         if not os.path.exists(self.cloneDestination):
3606             os.makedirs(self.cloneDestination)
3607         chdir(self.cloneDestination)
3608
3609         init_cmd = [ "git", "init" ]
3610         if self.cloneBare:
3611             init_cmd.append("--bare")
3612         retcode = subprocess.call(init_cmd)
3613         if retcode:
3614             raise CalledProcessError(retcode, init_cmd)
3615
3616         if not P4Sync.run(self, depotPaths):
3617             return False
3618
3619         # create a master branch and check out a work tree
3620         if gitBranchExists(self.branch):
3621             system([ "git", "branch", "master", self.branch ])
3622             if not self.cloneBare:
3623                 system([ "git", "checkout", "-f" ])
3624         else:
3625             print 'Not checking out any branch, use ' \
3626                   '"git checkout -q -b master <branch>"'
3627
3628         # auto-set this variable if invoked with --use-client-spec
3629         if self.useClientSpec_from_options:
3630             system("git config --bool git-p4.useclientspec true")
3631
3632         return True
3633
3634 class P4Branches(Command):
3635     def __init__(self):
3636         Command.__init__(self)
3637         self.options = [ ]
3638         self.description = ("Shows the git branches that hold imports and their "
3639                             + "corresponding perforce depot paths")
3640         self.verbose = False
3641
3642     def run(self, args):
3643         if originP4BranchesExist():
3644             createOrUpdateBranchesFromOrigin()
3645
3646         cmdline = "git rev-parse --symbolic "
3647         cmdline += " --remotes"
3648
3649         for line in read_pipe_lines(cmdline):
3650             line = line.strip()
3651
3652             if not line.startswith('p4/') or line == "p4/HEAD":
3653                 continue
3654             branch = line
3655
3656             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3657             settings = extractSettingsGitLog(log)
3658
3659             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3660         return True
3661
3662 class HelpFormatter(optparse.IndentedHelpFormatter):
3663     def __init__(self):
3664         optparse.IndentedHelpFormatter.__init__(self)
3665
3666     def format_description(self, description):
3667         if description:
3668             return description + "\n"
3669         else:
3670             return ""
3671
3672 def printUsage(commands):
3673     print "usage: %s <command> [options]" % sys.argv[0]
3674     print ""
3675     print "valid commands: %s" % ", ".join(commands)
3676     print ""
3677     print "Try %s <command> --help for command specific help." % sys.argv[0]
3678     print ""
3679
3680 commands = {
3681     "debug" : P4Debug,
3682     "submit" : P4Submit,
3683     "commit" : P4Submit,
3684     "sync" : P4Sync,
3685     "rebase" : P4Rebase,
3686     "clone" : P4Clone,
3687     "rollback" : P4RollBack,
3688     "branches" : P4Branches
3689 }
3690
3691
3692 def main():
3693     if len(sys.argv[1:]) == 0:
3694         printUsage(commands.keys())
3695         sys.exit(2)
3696
3697     cmdName = sys.argv[1]
3698     try:
3699         klass = commands[cmdName]
3700         cmd = klass()
3701     except KeyError:
3702         print "unknown command %s" % cmdName
3703         print ""
3704         printUsage(commands.keys())
3705         sys.exit(2)
3706
3707     options = cmd.options
3708     cmd.gitdir = os.environ.get("GIT_DIR", None)
3709
3710     args = sys.argv[2:]
3711
3712     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3713     if cmd.needsGit:
3714         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3715
3716     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3717                                    options,
3718                                    description = cmd.description,
3719                                    formatter = HelpFormatter())
3720
3721     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3722     global verbose
3723     verbose = cmd.verbose
3724     if cmd.needsGit:
3725         if cmd.gitdir == None:
3726             cmd.gitdir = os.path.abspath(".git")
3727             if not isValidGitDir(cmd.gitdir):
3728                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3729                 if os.path.exists(cmd.gitdir):
3730                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3731                     if len(cdup) > 0:
3732                         chdir(cdup);
3733
3734         if not isValidGitDir(cmd.gitdir):
3735             if isValidGitDir(cmd.gitdir + "/.git"):
3736                 cmd.gitdir += "/.git"
3737             else:
3738                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3739
3740         os.environ["GIT_DIR"] = cmd.gitdir
3741
3742     if not cmd.run(args):
3743         parser.print_help()
3744         sys.exit(2)
3745
3746
3747 if __name__ == '__main__':
3748     main()