Merge branch 'jk/index-pack-wo-repo-from-stdin'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28
29 try:
30     from subprocess import CalledProcessError
31 except ImportError:
32     # from python2.7:subprocess.py
33     # Exception classes used by this module.
34     class CalledProcessError(Exception):
35         """This exception is raised when a process run by check_call() returns
36         a non-zero exit status.  The exit status will be stored in the
37         returncode attribute."""
38         def __init__(self, returncode, cmd):
39             self.returncode = returncode
40             self.cmd = cmd
41         def __str__(self):
42             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
44 verbose = False
45
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
48
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
51
52 def p4_build_cmd(cmd):
53     """Build a suitable p4 command line.
54
55     This consolidates building and returning a p4 command line into one
56     location. It means that hooking into the environment, or other configuration
57     can be done more easily.
58     """
59     real_cmd = ["p4"]
60
61     user = gitConfig("git-p4.user")
62     if len(user) > 0:
63         real_cmd += ["-u",user]
64
65     password = gitConfig("git-p4.password")
66     if len(password) > 0:
67         real_cmd += ["-P", password]
68
69     port = gitConfig("git-p4.port")
70     if len(port) > 0:
71         real_cmd += ["-p", port]
72
73     host = gitConfig("git-p4.host")
74     if len(host) > 0:
75         real_cmd += ["-H", host]
76
77     client = gitConfig("git-p4.client")
78     if len(client) > 0:
79         real_cmd += ["-c", client]
80
81     retries = gitConfigInt("git-p4.retries")
82     if retries is None:
83         # Perform 3 retries by default
84         retries = 3
85     real_cmd += ["-r", str(retries)]
86
87     if isinstance(cmd,basestring):
88         real_cmd = ' '.join(real_cmd) + ' ' + cmd
89     else:
90         real_cmd += cmd
91     return real_cmd
92
93 def git_dir(path):
94     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
95         This won't automatically add ".git" to a directory.
96     """
97     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
98     if not d or len(d) == 0:
99         return None
100     else:
101         return d
102
103 def chdir(path, is_client_path=False):
104     """Do chdir to the given path, and set the PWD environment
105        variable for use by P4.  It does not look at getcwd() output.
106        Since we're not using the shell, it is necessary to set the
107        PWD environment variable explicitly.
108
109        Normally, expand the path to force it to be absolute.  This
110        addresses the use of relative path names inside P4 settings,
111        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
112        as given; it looks for .p4config using PWD.
113
114        If is_client_path, the path was handed to us directly by p4,
115        and may be a symbolic link.  Do not call os.getcwd() in this
116        case, because it will cause p4 to think that PWD is not inside
117        the client path.
118        """
119
120     os.chdir(path)
121     if not is_client_path:
122         path = os.getcwd()
123     os.environ['PWD'] = path
124
125 def calcDiskFree():
126     """Return free space in bytes on the disk of the given dirname."""
127     if platform.system() == 'Windows':
128         free_bytes = ctypes.c_ulonglong(0)
129         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
130         return free_bytes.value
131     else:
132         st = os.statvfs(os.getcwd())
133         return st.f_bavail * st.f_frsize
134
135 def die(msg):
136     if verbose:
137         raise Exception(msg)
138     else:
139         sys.stderr.write(msg + "\n")
140         sys.exit(1)
141
142 def write_pipe(c, stdin):
143     if verbose:
144         sys.stderr.write('Writing pipe: %s\n' % str(c))
145
146     expand = isinstance(c,basestring)
147     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
148     pipe = p.stdin
149     val = pipe.write(stdin)
150     pipe.close()
151     if p.wait():
152         die('Command failed: %s' % str(c))
153
154     return val
155
156 def p4_write_pipe(c, stdin):
157     real_cmd = p4_build_cmd(c)
158     return write_pipe(real_cmd, stdin)
159
160 def read_pipe(c, ignore_error=False):
161     if verbose:
162         sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164     expand = isinstance(c,basestring)
165     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
166     (out, err) = p.communicate()
167     if p.returncode != 0 and not ignore_error:
168         die('Command failed: %s\nError: %s' % (str(c), err))
169     return out
170
171 def p4_read_pipe(c, ignore_error=False):
172     real_cmd = p4_build_cmd(c)
173     return read_pipe(real_cmd, ignore_error)
174
175 def read_pipe_lines(c):
176     if verbose:
177         sys.stderr.write('Reading pipe: %s\n' % str(c))
178
179     expand = isinstance(c, basestring)
180     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
181     pipe = p.stdout
182     val = pipe.readlines()
183     if pipe.close() or p.wait():
184         die('Command failed: %s' % str(c))
185
186     return val
187
188 def p4_read_pipe_lines(c):
189     """Specifically invoke p4 on the command supplied. """
190     real_cmd = p4_build_cmd(c)
191     return read_pipe_lines(real_cmd)
192
193 def p4_has_command(cmd):
194     """Ask p4 for help on this command.  If it returns an error, the
195        command does not exist in this version of p4."""
196     real_cmd = p4_build_cmd(["help", cmd])
197     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
198                                    stderr=subprocess.PIPE)
199     p.communicate()
200     return p.returncode == 0
201
202 def p4_has_move_command():
203     """See if the move command exists, that it supports -k, and that
204        it has not been administratively disabled.  The arguments
205        must be correct, but the filenames do not have to exist.  Use
206        ones with wildcards so even if they exist, it will fail."""
207
208     if not p4_has_command("move"):
209         return False
210     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
211     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
212     (out, err) = p.communicate()
213     # return code will be 1 in either case
214     if err.find("Invalid option") >= 0:
215         return False
216     if err.find("disabled") >= 0:
217         return False
218     # assume it failed because @... was invalid changelist
219     return True
220
221 def system(cmd, ignore_error=False):
222     expand = isinstance(cmd,basestring)
223     if verbose:
224         sys.stderr.write("executing %s\n" % str(cmd))
225     retcode = subprocess.call(cmd, shell=expand)
226     if retcode and not ignore_error:
227         raise CalledProcessError(retcode, cmd)
228
229     return retcode
230
231 def p4_system(cmd):
232     """Specifically invoke p4 as the system command. """
233     real_cmd = p4_build_cmd(cmd)
234     expand = isinstance(real_cmd, basestring)
235     retcode = subprocess.call(real_cmd, shell=expand)
236     if retcode:
237         raise CalledProcessError(retcode, real_cmd)
238
239 _p4_version_string = None
240 def p4_version_string():
241     """Read the version string, showing just the last line, which
242        hopefully is the interesting version bit.
243
244        $ p4 -V
245        Perforce - The Fast Software Configuration Management System.
246        Copyright 1995-2011 Perforce Software.  All rights reserved.
247        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
248     """
249     global _p4_version_string
250     if not _p4_version_string:
251         a = p4_read_pipe_lines(["-V"])
252         _p4_version_string = a[-1].rstrip()
253     return _p4_version_string
254
255 def p4_integrate(src, dest):
256     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
257
258 def p4_sync(f, *options):
259     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
260
261 def p4_add(f):
262     # forcibly add file names with wildcards
263     if wildcard_present(f):
264         p4_system(["add", "-f", f])
265     else:
266         p4_system(["add", f])
267
268 def p4_delete(f):
269     p4_system(["delete", wildcard_encode(f)])
270
271 def p4_edit(f, *options):
272     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
273
274 def p4_revert(f):
275     p4_system(["revert", wildcard_encode(f)])
276
277 def p4_reopen(type, f):
278     p4_system(["reopen", "-t", type, wildcard_encode(f)])
279
280 def p4_reopen_in_change(changelist, files):
281     cmd = ["reopen", "-c", str(changelist)] + files
282     p4_system(cmd)
283
284 def p4_move(src, dest):
285     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
286
287 def p4_last_change():
288     results = p4CmdList(["changes", "-m", "1"])
289     return int(results[0]['change'])
290
291 def p4_describe(change):
292     """Make sure it returns a valid result by checking for
293        the presence of field "time".  Return a dict of the
294        results."""
295
296     ds = p4CmdList(["describe", "-s", str(change)])
297     if len(ds) != 1:
298         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
299
300     d = ds[0]
301
302     if "p4ExitCode" in d:
303         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
304                                                       str(d)))
305     if "code" in d:
306         if d["code"] == "error":
307             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
308
309     if "time" not in d:
310         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
311
312     return d
313
314 #
315 # Canonicalize the p4 type and return a tuple of the
316 # base type, plus any modifiers.  See "p4 help filetypes"
317 # for a list and explanation.
318 #
319 def split_p4_type(p4type):
320
321     p4_filetypes_historical = {
322         "ctempobj": "binary+Sw",
323         "ctext": "text+C",
324         "cxtext": "text+Cx",
325         "ktext": "text+k",
326         "kxtext": "text+kx",
327         "ltext": "text+F",
328         "tempobj": "binary+FSw",
329         "ubinary": "binary+F",
330         "uresource": "resource+F",
331         "uxbinary": "binary+Fx",
332         "xbinary": "binary+x",
333         "xltext": "text+Fx",
334         "xtempobj": "binary+Swx",
335         "xtext": "text+x",
336         "xunicode": "unicode+x",
337         "xutf16": "utf16+x",
338     }
339     if p4type in p4_filetypes_historical:
340         p4type = p4_filetypes_historical[p4type]
341     mods = ""
342     s = p4type.split("+")
343     base = s[0]
344     mods = ""
345     if len(s) > 1:
346         mods = s[1]
347     return (base, mods)
348
349 #
350 # return the raw p4 type of a file (text, text+ko, etc)
351 #
352 def p4_type(f):
353     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
354     return results[0]['headType']
355
356 #
357 # Given a type base and modifier, return a regexp matching
358 # the keywords that can be expanded in the file
359 #
360 def p4_keywords_regexp_for_type(base, type_mods):
361     if base in ("text", "unicode", "binary"):
362         kwords = None
363         if "ko" in type_mods:
364             kwords = 'Id|Header'
365         elif "k" in type_mods:
366             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
367         else:
368             return None
369         pattern = r"""
370             \$              # Starts with a dollar, followed by...
371             (%s)            # one of the keywords, followed by...
372             (:[^$\n]+)?     # possibly an old expansion, followed by...
373             \$              # another dollar
374             """ % kwords
375         return pattern
376     else:
377         return None
378
379 #
380 # Given a file, return a regexp matching the possible
381 # RCS keywords that will be expanded, or None for files
382 # with kw expansion turned off.
383 #
384 def p4_keywords_regexp_for_file(file):
385     if not os.path.exists(file):
386         return None
387     else:
388         (type_base, type_mods) = split_p4_type(p4_type(file))
389         return p4_keywords_regexp_for_type(type_base, type_mods)
390
391 def setP4ExecBit(file, mode):
392     # Reopens an already open file and changes the execute bit to match
393     # the execute bit setting in the passed in mode.
394
395     p4Type = "+x"
396
397     if not isModeExec(mode):
398         p4Type = getP4OpenedType(file)
399         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
400         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
401         if p4Type[-1] == "+":
402             p4Type = p4Type[0:-1]
403
404     p4_reopen(p4Type, file)
405
406 def getP4OpenedType(file):
407     # Returns the perforce file type for the given file.
408
409     result = p4_read_pipe(["opened", wildcard_encode(file)])
410     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
411     if match:
412         return match.group(1)
413     else:
414         die("Could not determine file type for %s (result: '%s')" % (file, result))
415
416 # Return the set of all p4 labels
417 def getP4Labels(depotPaths):
418     labels = set()
419     if isinstance(depotPaths,basestring):
420         depotPaths = [depotPaths]
421
422     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
423         label = l['label']
424         labels.add(label)
425
426     return labels
427
428 # Return the set of all git tags
429 def getGitTags():
430     gitTags = set()
431     for line in read_pipe_lines(["git", "tag"]):
432         tag = line.strip()
433         gitTags.add(tag)
434     return gitTags
435
436 def diffTreePattern():
437     # This is a simple generator for the diff tree regex pattern. This could be
438     # a class variable if this and parseDiffTreeEntry were a part of a class.
439     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
440     while True:
441         yield pattern
442
443 def parseDiffTreeEntry(entry):
444     """Parses a single diff tree entry into its component elements.
445
446     See git-diff-tree(1) manpage for details about the format of the diff
447     output. This method returns a dictionary with the following elements:
448
449     src_mode - The mode of the source file
450     dst_mode - The mode of the destination file
451     src_sha1 - The sha1 for the source file
452     dst_sha1 - The sha1 fr the destination file
453     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
454     status_score - The score for the status (applicable for 'C' and 'R'
455                    statuses). This is None if there is no score.
456     src - The path for the source file.
457     dst - The path for the destination file. This is only present for
458           copy or renames. If it is not present, this is None.
459
460     If the pattern is not matched, None is returned."""
461
462     match = diffTreePattern().next().match(entry)
463     if match:
464         return {
465             'src_mode': match.group(1),
466             'dst_mode': match.group(2),
467             'src_sha1': match.group(3),
468             'dst_sha1': match.group(4),
469             'status': match.group(5),
470             'status_score': match.group(6),
471             'src': match.group(7),
472             'dst': match.group(10)
473         }
474     return None
475
476 def isModeExec(mode):
477     # Returns True if the given git mode represents an executable file,
478     # otherwise False.
479     return mode[-3:] == "755"
480
481 def isModeExecChanged(src_mode, dst_mode):
482     return isModeExec(src_mode) != isModeExec(dst_mode)
483
484 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
485
486     if isinstance(cmd,basestring):
487         cmd = "-G " + cmd
488         expand = True
489     else:
490         cmd = ["-G"] + cmd
491         expand = False
492
493     cmd = p4_build_cmd(cmd)
494     if verbose:
495         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
496
497     # Use a temporary file to avoid deadlocks without
498     # subprocess.communicate(), which would put another copy
499     # of stdout into memory.
500     stdin_file = None
501     if stdin is not None:
502         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
503         if isinstance(stdin,basestring):
504             stdin_file.write(stdin)
505         else:
506             for i in stdin:
507                 stdin_file.write(i + '\n')
508         stdin_file.flush()
509         stdin_file.seek(0)
510
511     p4 = subprocess.Popen(cmd,
512                           shell=expand,
513                           stdin=stdin_file,
514                           stdout=subprocess.PIPE)
515
516     result = []
517     try:
518         while True:
519             entry = marshal.load(p4.stdout)
520             if cb is not None:
521                 cb(entry)
522             else:
523                 result.append(entry)
524     except EOFError:
525         pass
526     exitCode = p4.wait()
527     if exitCode != 0:
528         entry = {}
529         entry["p4ExitCode"] = exitCode
530         result.append(entry)
531
532     return result
533
534 def p4Cmd(cmd):
535     list = p4CmdList(cmd)
536     result = {}
537     for entry in list:
538         result.update(entry)
539     return result;
540
541 def p4Where(depotPath):
542     if not depotPath.endswith("/"):
543         depotPath += "/"
544     depotPathLong = depotPath + "..."
545     outputList = p4CmdList(["where", depotPathLong])
546     output = None
547     for entry in outputList:
548         if "depotFile" in entry:
549             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
550             # The base path always ends with "/...".
551             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
552                 output = entry
553                 break
554         elif "data" in entry:
555             data = entry.get("data")
556             space = data.find(" ")
557             if data[:space] == depotPath:
558                 output = entry
559                 break
560     if output == None:
561         return ""
562     if output["code"] == "error":
563         return ""
564     clientPath = ""
565     if "path" in output:
566         clientPath = output.get("path")
567     elif "data" in output:
568         data = output.get("data")
569         lastSpace = data.rfind(" ")
570         clientPath = data[lastSpace + 1:]
571
572     if clientPath.endswith("..."):
573         clientPath = clientPath[:-3]
574     return clientPath
575
576 def currentGitBranch():
577     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
578     if retcode != 0:
579         # on a detached head
580         return None
581     else:
582         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
583
584 def isValidGitDir(path):
585     return git_dir(path) != None
586
587 def parseRevision(ref):
588     return read_pipe("git rev-parse %s" % ref).strip()
589
590 def branchExists(ref):
591     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
592                      ignore_error=True)
593     return len(rev) > 0
594
595 def extractLogMessageFromGitCommit(commit):
596     logMessage = ""
597
598     ## fixme: title is first line of commit, not 1st paragraph.
599     foundTitle = False
600     for log in read_pipe_lines("git cat-file commit %s" % commit):
601        if not foundTitle:
602            if len(log) == 1:
603                foundTitle = True
604            continue
605
606        logMessage += log
607     return logMessage
608
609 def extractSettingsGitLog(log):
610     values = {}
611     for line in log.split("\n"):
612         line = line.strip()
613         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
614         if not m:
615             continue
616
617         assignments = m.group(1).split (':')
618         for a in assignments:
619             vals = a.split ('=')
620             key = vals[0].strip()
621             val = ('='.join (vals[1:])).strip()
622             if val.endswith ('\"') and val.startswith('"'):
623                 val = val[1:-1]
624
625             values[key] = val
626
627     paths = values.get("depot-paths")
628     if not paths:
629         paths = values.get("depot-path")
630     if paths:
631         values['depot-paths'] = paths.split(',')
632     return values
633
634 def gitBranchExists(branch):
635     proc = subprocess.Popen(["git", "rev-parse", branch],
636                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
637     return proc.wait() == 0;
638
639 _gitConfig = {}
640
641 def gitConfig(key, typeSpecifier=None):
642     if not _gitConfig.has_key(key):
643         cmd = [ "git", "config" ]
644         if typeSpecifier:
645             cmd += [ typeSpecifier ]
646         cmd += [ key ]
647         s = read_pipe(cmd, ignore_error=True)
648         _gitConfig[key] = s.strip()
649     return _gitConfig[key]
650
651 def gitConfigBool(key):
652     """Return a bool, using git config --bool.  It is True only if the
653        variable is set to true, and False if set to false or not present
654        in the config."""
655
656     if not _gitConfig.has_key(key):
657         _gitConfig[key] = gitConfig(key, '--bool') == "true"
658     return _gitConfig[key]
659
660 def gitConfigInt(key):
661     if not _gitConfig.has_key(key):
662         cmd = [ "git", "config", "--int", key ]
663         s = read_pipe(cmd, ignore_error=True)
664         v = s.strip()
665         try:
666             _gitConfig[key] = int(gitConfig(key, '--int'))
667         except ValueError:
668             _gitConfig[key] = None
669     return _gitConfig[key]
670
671 def gitConfigList(key):
672     if not _gitConfig.has_key(key):
673         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
674         _gitConfig[key] = s.strip().split(os.linesep)
675         if _gitConfig[key] == ['']:
676             _gitConfig[key] = []
677     return _gitConfig[key]
678
679 def p4BranchesInGit(branchesAreInRemotes=True):
680     """Find all the branches whose names start with "p4/", looking
681        in remotes or heads as specified by the argument.  Return
682        a dictionary of { branch: revision } for each one found.
683        The branch names are the short names, without any
684        "p4/" prefix."""
685
686     branches = {}
687
688     cmdline = "git rev-parse --symbolic "
689     if branchesAreInRemotes:
690         cmdline += "--remotes"
691     else:
692         cmdline += "--branches"
693
694     for line in read_pipe_lines(cmdline):
695         line = line.strip()
696
697         # only import to p4/
698         if not line.startswith('p4/'):
699             continue
700         # special symbolic ref to p4/master
701         if line == "p4/HEAD":
702             continue
703
704         # strip off p4/ prefix
705         branch = line[len("p4/"):]
706
707         branches[branch] = parseRevision(line)
708
709     return branches
710
711 def branch_exists(branch):
712     """Make sure that the given ref name really exists."""
713
714     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
715     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
716     out, _ = p.communicate()
717     if p.returncode:
718         return False
719     # expect exactly one line of output: the branch name
720     return out.rstrip() == branch
721
722 def findUpstreamBranchPoint(head = "HEAD"):
723     branches = p4BranchesInGit()
724     # map from depot-path to branch name
725     branchByDepotPath = {}
726     for branch in branches.keys():
727         tip = branches[branch]
728         log = extractLogMessageFromGitCommit(tip)
729         settings = extractSettingsGitLog(log)
730         if settings.has_key("depot-paths"):
731             paths = ",".join(settings["depot-paths"])
732             branchByDepotPath[paths] = "remotes/p4/" + branch
733
734     settings = None
735     parent = 0
736     while parent < 65535:
737         commit = head + "~%s" % parent
738         log = extractLogMessageFromGitCommit(commit)
739         settings = extractSettingsGitLog(log)
740         if settings.has_key("depot-paths"):
741             paths = ",".join(settings["depot-paths"])
742             if branchByDepotPath.has_key(paths):
743                 return [branchByDepotPath[paths], settings]
744
745         parent = parent + 1
746
747     return ["", settings]
748
749 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
750     if not silent:
751         print ("Creating/updating branch(es) in %s based on origin branch(es)"
752                % localRefPrefix)
753
754     originPrefix = "origin/p4/"
755
756     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
757         line = line.strip()
758         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
759             continue
760
761         headName = line[len(originPrefix):]
762         remoteHead = localRefPrefix + headName
763         originHead = line
764
765         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
766         if (not original.has_key('depot-paths')
767             or not original.has_key('change')):
768             continue
769
770         update = False
771         if not gitBranchExists(remoteHead):
772             if verbose:
773                 print "creating %s" % remoteHead
774             update = True
775         else:
776             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
777             if settings.has_key('change') > 0:
778                 if settings['depot-paths'] == original['depot-paths']:
779                     originP4Change = int(original['change'])
780                     p4Change = int(settings['change'])
781                     if originP4Change > p4Change:
782                         print ("%s (%s) is newer than %s (%s). "
783                                "Updating p4 branch from origin."
784                                % (originHead, originP4Change,
785                                   remoteHead, p4Change))
786                         update = True
787                 else:
788                     print ("Ignoring: %s was imported from %s while "
789                            "%s was imported from %s"
790                            % (originHead, ','.join(original['depot-paths']),
791                               remoteHead, ','.join(settings['depot-paths'])))
792
793         if update:
794             system("git update-ref %s %s" % (remoteHead, originHead))
795
796 def originP4BranchesExist():
797         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
798
799
800 def p4ParseNumericChangeRange(parts):
801     changeStart = int(parts[0][1:])
802     if parts[1] == '#head':
803         changeEnd = p4_last_change()
804     else:
805         changeEnd = int(parts[1])
806
807     return (changeStart, changeEnd)
808
809 def chooseBlockSize(blockSize):
810     if blockSize:
811         return blockSize
812     else:
813         return defaultBlockSize
814
815 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
816     assert depotPaths
817
818     # Parse the change range into start and end. Try to find integer
819     # revision ranges as these can be broken up into blocks to avoid
820     # hitting server-side limits (maxrows, maxscanresults). But if
821     # that doesn't work, fall back to using the raw revision specifier
822     # strings, without using block mode.
823
824     if changeRange is None or changeRange == '':
825         changeStart = 1
826         changeEnd = p4_last_change()
827         block_size = chooseBlockSize(requestedBlockSize)
828     else:
829         parts = changeRange.split(',')
830         assert len(parts) == 2
831         try:
832             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
833             block_size = chooseBlockSize(requestedBlockSize)
834         except:
835             changeStart = parts[0][1:]
836             changeEnd = parts[1]
837             if requestedBlockSize:
838                 die("cannot use --changes-block-size with non-numeric revisions")
839             block_size = None
840
841     changes = []
842
843     # Retrieve changes a block at a time, to prevent running
844     # into a MaxResults/MaxScanRows error from the server.
845
846     while True:
847         cmd = ['changes']
848
849         if block_size:
850             end = min(changeEnd, changeStart + block_size)
851             revisionRange = "%d,%d" % (changeStart, end)
852         else:
853             revisionRange = "%s,%s" % (changeStart, changeEnd)
854
855         for p in depotPaths:
856             cmd += ["%s...@%s" % (p, revisionRange)]
857
858         # Insert changes in chronological order
859         for line in reversed(p4_read_pipe_lines(cmd)):
860             changes.append(int(line.split(" ")[1]))
861
862         if not block_size:
863             break
864
865         if end >= changeEnd:
866             break
867
868         changeStart = end + 1
869
870     changes = sorted(changes)
871     return changes
872
873 def p4PathStartsWith(path, prefix):
874     # This method tries to remedy a potential mixed-case issue:
875     #
876     # If UserA adds  //depot/DirA/file1
877     # and UserB adds //depot/dira/file2
878     #
879     # we may or may not have a problem. If you have core.ignorecase=true,
880     # we treat DirA and dira as the same directory
881     if gitConfigBool("core.ignorecase"):
882         return path.lower().startswith(prefix.lower())
883     return path.startswith(prefix)
884
885 def getClientSpec():
886     """Look at the p4 client spec, create a View() object that contains
887        all the mappings, and return it."""
888
889     specList = p4CmdList("client -o")
890     if len(specList) != 1:
891         die('Output from "client -o" is %d lines, expecting 1' %
892             len(specList))
893
894     # dictionary of all client parameters
895     entry = specList[0]
896
897     # the //client/ name
898     client_name = entry["Client"]
899
900     # just the keys that start with "View"
901     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
902
903     # hold this new View
904     view = View(client_name)
905
906     # append the lines, in order, to the view
907     for view_num in range(len(view_keys)):
908         k = "View%d" % view_num
909         if k not in view_keys:
910             die("Expected view key %s missing" % k)
911         view.append(entry[k])
912
913     return view
914
915 def getClientRoot():
916     """Grab the client directory."""
917
918     output = p4CmdList("client -o")
919     if len(output) != 1:
920         die('Output from "client -o" is %d lines, expecting 1' % len(output))
921
922     entry = output[0]
923     if "Root" not in entry:
924         die('Client has no "Root"')
925
926     return entry["Root"]
927
928 #
929 # P4 wildcards are not allowed in filenames.  P4 complains
930 # if you simply add them, but you can force it with "-f", in
931 # which case it translates them into %xx encoding internally.
932 #
933 def wildcard_decode(path):
934     # Search for and fix just these four characters.  Do % last so
935     # that fixing it does not inadvertently create new %-escapes.
936     # Cannot have * in a filename in windows; untested as to
937     # what p4 would do in such a case.
938     if not platform.system() == "Windows":
939         path = path.replace("%2A", "*")
940     path = path.replace("%23", "#") \
941                .replace("%40", "@") \
942                .replace("%25", "%")
943     return path
944
945 def wildcard_encode(path):
946     # do % first to avoid double-encoding the %s introduced here
947     path = path.replace("%", "%25") \
948                .replace("*", "%2A") \
949                .replace("#", "%23") \
950                .replace("@", "%40")
951     return path
952
953 def wildcard_present(path):
954     m = re.search("[*#@%]", path)
955     return m is not None
956
957 class LargeFileSystem(object):
958     """Base class for large file system support."""
959
960     def __init__(self, writeToGitStream):
961         self.largeFiles = set()
962         self.writeToGitStream = writeToGitStream
963
964     def generatePointer(self, cloneDestination, contentFile):
965         """Return the content of a pointer file that is stored in Git instead of
966            the actual content."""
967         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
968
969     def pushFile(self, localLargeFile):
970         """Push the actual content which is not stored in the Git repository to
971            a server."""
972         assert False, "Method 'pushFile' required in " + self.__class__.__name__
973
974     def hasLargeFileExtension(self, relPath):
975         return reduce(
976             lambda a, b: a or b,
977             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
978             False
979         )
980
981     def generateTempFile(self, contents):
982         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983         for d in contents:
984             contentFile.write(d)
985         contentFile.close()
986         return contentFile.name
987
988     def exceedsLargeFileThreshold(self, relPath, contents):
989         if gitConfigInt('git-p4.largeFileThreshold'):
990             contentsSize = sum(len(d) for d in contents)
991             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
992                 return True
993         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
994             contentsSize = sum(len(d) for d in contents)
995             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
996                 return False
997             contentTempFile = self.generateTempFile(contents)
998             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
999             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1000             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1001             zf.close()
1002             compressedContentsSize = zf.infolist()[0].compress_size
1003             os.remove(contentTempFile)
1004             os.remove(compressedContentFile.name)
1005             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1006                 return True
1007         return False
1008
1009     def addLargeFile(self, relPath):
1010         self.largeFiles.add(relPath)
1011
1012     def removeLargeFile(self, relPath):
1013         self.largeFiles.remove(relPath)
1014
1015     def isLargeFile(self, relPath):
1016         return relPath in self.largeFiles
1017
1018     def processContent(self, git_mode, relPath, contents):
1019         """Processes the content of git fast import. This method decides if a
1020            file is stored in the large file system and handles all necessary
1021            steps."""
1022         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1023             contentTempFile = self.generateTempFile(contents)
1024             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1025             if pointer_git_mode:
1026                 git_mode = pointer_git_mode
1027             if localLargeFile:
1028                 # Move temp file to final location in large file system
1029                 largeFileDir = os.path.dirname(localLargeFile)
1030                 if not os.path.isdir(largeFileDir):
1031                     os.makedirs(largeFileDir)
1032                 shutil.move(contentTempFile, localLargeFile)
1033                 self.addLargeFile(relPath)
1034                 if gitConfigBool('git-p4.largeFilePush'):
1035                     self.pushFile(localLargeFile)
1036                 if verbose:
1037                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1038         return (git_mode, contents)
1039
1040 class MockLFS(LargeFileSystem):
1041     """Mock large file system for testing."""
1042
1043     def generatePointer(self, contentFile):
1044         """The pointer content is the original content prefixed with "pointer-".
1045            The local filename of the large file storage is derived from the file content.
1046            """
1047         with open(contentFile, 'r') as f:
1048             content = next(f)
1049             gitMode = '100644'
1050             pointerContents = 'pointer-' + content
1051             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1052             return (gitMode, pointerContents, localLargeFile)
1053
1054     def pushFile(self, localLargeFile):
1055         """The remote filename of the large file storage is the same as the local
1056            one but in a different directory.
1057            """
1058         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1059         if not os.path.exists(remotePath):
1060             os.makedirs(remotePath)
1061         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1062
1063 class GitLFS(LargeFileSystem):
1064     """Git LFS as backend for the git-p4 large file system.
1065        See https://git-lfs.github.com/ for details."""
1066
1067     def __init__(self, *args):
1068         LargeFileSystem.__init__(self, *args)
1069         self.baseGitAttributes = []
1070
1071     def generatePointer(self, contentFile):
1072         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1073            mode and content which is stored in the Git repository instead of
1074            the actual content. Return also the new location of the actual
1075            content.
1076            """
1077         if os.path.getsize(contentFile) == 0:
1078             return (None, '', None)
1079
1080         pointerProcess = subprocess.Popen(
1081             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1082             stdout=subprocess.PIPE
1083         )
1084         pointerFile = pointerProcess.stdout.read()
1085         if pointerProcess.wait():
1086             os.remove(contentFile)
1087             die('git-lfs pointer command failed. Did you install the extension?')
1088
1089         # Git LFS removed the preamble in the output of the 'pointer' command
1090         # starting from version 1.2.0. Check for the preamble here to support
1091         # earlier versions.
1092         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1093         if pointerFile.startswith('Git LFS pointer for'):
1094             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1095
1096         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1097         localLargeFile = os.path.join(
1098             os.getcwd(),
1099             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1100             oid,
1101         )
1102         # LFS Spec states that pointer files should not have the executable bit set.
1103         gitMode = '100644'
1104         return (gitMode, pointerFile, localLargeFile)
1105
1106     def pushFile(self, localLargeFile):
1107         uploadProcess = subprocess.Popen(
1108             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1109         )
1110         if uploadProcess.wait():
1111             die('git-lfs push command failed. Did you define a remote?')
1112
1113     def generateGitAttributes(self):
1114         return (
1115             self.baseGitAttributes +
1116             [
1117                 '\n',
1118                 '#\n',
1119                 '# Git LFS (see https://git-lfs.github.com/)\n',
1120                 '#\n',
1121             ] +
1122             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1123                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1124             ] +
1125             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1126                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1127             ]
1128         )
1129
1130     def addLargeFile(self, relPath):
1131         LargeFileSystem.addLargeFile(self, relPath)
1132         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1133
1134     def removeLargeFile(self, relPath):
1135         LargeFileSystem.removeLargeFile(self, relPath)
1136         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1137
1138     def processContent(self, git_mode, relPath, contents):
1139         if relPath == '.gitattributes':
1140             self.baseGitAttributes = contents
1141             return (git_mode, self.generateGitAttributes())
1142         else:
1143             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1144
1145 class Command:
1146     def __init__(self):
1147         self.usage = "usage: %prog [options]"
1148         self.needsGit = True
1149         self.verbose = False
1150
1151 class P4UserMap:
1152     def __init__(self):
1153         self.userMapFromPerforceServer = False
1154         self.myP4UserId = None
1155
1156     def p4UserId(self):
1157         if self.myP4UserId:
1158             return self.myP4UserId
1159
1160         results = p4CmdList("user -o")
1161         for r in results:
1162             if r.has_key('User'):
1163                 self.myP4UserId = r['User']
1164                 return r['User']
1165         die("Could not find your p4 user id")
1166
1167     def p4UserIsMe(self, p4User):
1168         # return True if the given p4 user is actually me
1169         me = self.p4UserId()
1170         if not p4User or p4User != me:
1171             return False
1172         else:
1173             return True
1174
1175     def getUserCacheFilename(self):
1176         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1177         return home + "/.gitp4-usercache.txt"
1178
1179     def getUserMapFromPerforceServer(self):
1180         if self.userMapFromPerforceServer:
1181             return
1182         self.users = {}
1183         self.emails = {}
1184
1185         for output in p4CmdList("users"):
1186             if not output.has_key("User"):
1187                 continue
1188             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1189             self.emails[output["Email"]] = output["User"]
1190
1191         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1192         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1193             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1194             if mapUser and len(mapUser[0]) == 3:
1195                 user = mapUser[0][0]
1196                 fullname = mapUser[0][1]
1197                 email = mapUser[0][2]
1198                 self.users[user] = fullname + " <" + email + ">"
1199                 self.emails[email] = user
1200
1201         s = ''
1202         for (key, val) in self.users.items():
1203             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1204
1205         open(self.getUserCacheFilename(), "wb").write(s)
1206         self.userMapFromPerforceServer = True
1207
1208     def loadUserMapFromCache(self):
1209         self.users = {}
1210         self.userMapFromPerforceServer = False
1211         try:
1212             cache = open(self.getUserCacheFilename(), "rb")
1213             lines = cache.readlines()
1214             cache.close()
1215             for line in lines:
1216                 entry = line.strip().split("\t")
1217                 self.users[entry[0]] = entry[1]
1218         except IOError:
1219             self.getUserMapFromPerforceServer()
1220
1221 class P4Debug(Command):
1222     def __init__(self):
1223         Command.__init__(self)
1224         self.options = []
1225         self.description = "A tool to debug the output of p4 -G."
1226         self.needsGit = False
1227
1228     def run(self, args):
1229         j = 0
1230         for output in p4CmdList(args):
1231             print 'Element: %d' % j
1232             j += 1
1233             print output
1234         return True
1235
1236 class P4RollBack(Command):
1237     def __init__(self):
1238         Command.__init__(self)
1239         self.options = [
1240             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1241         ]
1242         self.description = "A tool to debug the multi-branch import. Don't use :)"
1243         self.rollbackLocalBranches = False
1244
1245     def run(self, args):
1246         if len(args) != 1:
1247             return False
1248         maxChange = int(args[0])
1249
1250         if "p4ExitCode" in p4Cmd("changes -m 1"):
1251             die("Problems executing p4");
1252
1253         if self.rollbackLocalBranches:
1254             refPrefix = "refs/heads/"
1255             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1256         else:
1257             refPrefix = "refs/remotes/"
1258             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1259
1260         for line in lines:
1261             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1262                 line = line.strip()
1263                 ref = refPrefix + line
1264                 log = extractLogMessageFromGitCommit(ref)
1265                 settings = extractSettingsGitLog(log)
1266
1267                 depotPaths = settings['depot-paths']
1268                 change = settings['change']
1269
1270                 changed = False
1271
1272                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1273                                                            for p in depotPaths]))) == 0:
1274                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1275                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1276                     continue
1277
1278                 while change and int(change) > maxChange:
1279                     changed = True
1280                     if self.verbose:
1281                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1282                     system("git update-ref %s \"%s^\"" % (ref, ref))
1283                     log = extractLogMessageFromGitCommit(ref)
1284                     settings =  extractSettingsGitLog(log)
1285
1286
1287                     depotPaths = settings['depot-paths']
1288                     change = settings['change']
1289
1290                 if changed:
1291                     print "%s rewound to %s" % (ref, change)
1292
1293         return True
1294
1295 class P4Submit(Command, P4UserMap):
1296
1297     conflict_behavior_choices = ("ask", "skip", "quit")
1298
1299     def __init__(self):
1300         Command.__init__(self)
1301         P4UserMap.__init__(self)
1302         self.options = [
1303                 optparse.make_option("--origin", dest="origin"),
1304                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1305                 # preserve the user, requires relevant p4 permissions
1306                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1307                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1308                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1309                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1310                 optparse.make_option("--conflict", dest="conflict_behavior",
1311                                      choices=self.conflict_behavior_choices),
1312                 optparse.make_option("--branch", dest="branch"),
1313                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1314                                      help="Shelve instead of submit. Shelved files are reverted, "
1315                                      "restoring the workspace to the state before the shelve"),
1316                 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1317                                      metavar="CHANGELIST",
1318                                      help="update an existing shelved changelist, implies --shelve")
1319         ]
1320         self.description = "Submit changes from git to the perforce depot."
1321         self.usage += " [name of git branch to submit into perforce depot]"
1322         self.origin = ""
1323         self.detectRenames = False
1324         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1325         self.dry_run = False
1326         self.shelve = False
1327         self.update_shelve = None
1328         self.prepare_p4_only = False
1329         self.conflict_behavior = None
1330         self.isWindows = (platform.system() == "Windows")
1331         self.exportLabels = False
1332         self.p4HasMoveCommand = p4_has_move_command()
1333         self.branch = None
1334
1335         if gitConfig('git-p4.largeFileSystem'):
1336             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1337
1338     def check(self):
1339         if len(p4CmdList("opened ...")) > 0:
1340             die("You have files opened with perforce! Close them before starting the sync.")
1341
1342     def separate_jobs_from_description(self, message):
1343         """Extract and return a possible Jobs field in the commit
1344            message.  It goes into a separate section in the p4 change
1345            specification.
1346
1347            A jobs line starts with "Jobs:" and looks like a new field
1348            in a form.  Values are white-space separated on the same
1349            line or on following lines that start with a tab.
1350
1351            This does not parse and extract the full git commit message
1352            like a p4 form.  It just sees the Jobs: line as a marker
1353            to pass everything from then on directly into the p4 form,
1354            but outside the description section.
1355
1356            Return a tuple (stripped log message, jobs string)."""
1357
1358         m = re.search(r'^Jobs:', message, re.MULTILINE)
1359         if m is None:
1360             return (message, None)
1361
1362         jobtext = message[m.start():]
1363         stripped_message = message[:m.start()].rstrip()
1364         return (stripped_message, jobtext)
1365
1366     def prepareLogMessage(self, template, message, jobs):
1367         """Edits the template returned from "p4 change -o" to insert
1368            the message in the Description field, and the jobs text in
1369            the Jobs field."""
1370         result = ""
1371
1372         inDescriptionSection = False
1373
1374         for line in template.split("\n"):
1375             if line.startswith("#"):
1376                 result += line + "\n"
1377                 continue
1378
1379             if inDescriptionSection:
1380                 if line.startswith("Files:") or line.startswith("Jobs:"):
1381                     inDescriptionSection = False
1382                     # insert Jobs section
1383                     if jobs:
1384                         result += jobs + "\n"
1385                 else:
1386                     continue
1387             else:
1388                 if line.startswith("Description:"):
1389                     inDescriptionSection = True
1390                     line += "\n"
1391                     for messageLine in message.split("\n"):
1392                         line += "\t" + messageLine + "\n"
1393
1394             result += line + "\n"
1395
1396         return result
1397
1398     def patchRCSKeywords(self, file, pattern):
1399         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1400         (handle, outFileName) = tempfile.mkstemp(dir='.')
1401         try:
1402             outFile = os.fdopen(handle, "w+")
1403             inFile = open(file, "r")
1404             regexp = re.compile(pattern, re.VERBOSE)
1405             for line in inFile.readlines():
1406                 line = regexp.sub(r'$\1$', line)
1407                 outFile.write(line)
1408             inFile.close()
1409             outFile.close()
1410             # Forcibly overwrite the original file
1411             os.unlink(file)
1412             shutil.move(outFileName, file)
1413         except:
1414             # cleanup our temporary file
1415             os.unlink(outFileName)
1416             print "Failed to strip RCS keywords in %s" % file
1417             raise
1418
1419         print "Patched up RCS keywords in %s" % file
1420
1421     def p4UserForCommit(self,id):
1422         # Return the tuple (perforce user,git email) for a given git commit id
1423         self.getUserMapFromPerforceServer()
1424         gitEmail = read_pipe(["git", "log", "--max-count=1",
1425                               "--format=%ae", id])
1426         gitEmail = gitEmail.strip()
1427         if not self.emails.has_key(gitEmail):
1428             return (None,gitEmail)
1429         else:
1430             return (self.emails[gitEmail],gitEmail)
1431
1432     def checkValidP4Users(self,commits):
1433         # check if any git authors cannot be mapped to p4 users
1434         for id in commits:
1435             (user,email) = self.p4UserForCommit(id)
1436             if not user:
1437                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1438                 if gitConfigBool("git-p4.allowMissingP4Users"):
1439                     print "%s" % msg
1440                 else:
1441                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1442
1443     def lastP4Changelist(self):
1444         # Get back the last changelist number submitted in this client spec. This
1445         # then gets used to patch up the username in the change. If the same
1446         # client spec is being used by multiple processes then this might go
1447         # wrong.
1448         results = p4CmdList("client -o")        # find the current client
1449         client = None
1450         for r in results:
1451             if r.has_key('Client'):
1452                 client = r['Client']
1453                 break
1454         if not client:
1455             die("could not get client spec")
1456         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1457         for r in results:
1458             if r.has_key('change'):
1459                 return r['change']
1460         die("Could not get changelist number for last submit - cannot patch up user details")
1461
1462     def modifyChangelistUser(self, changelist, newUser):
1463         # fixup the user field of a changelist after it has been submitted.
1464         changes = p4CmdList("change -o %s" % changelist)
1465         if len(changes) != 1:
1466             die("Bad output from p4 change modifying %s to user %s" %
1467                 (changelist, newUser))
1468
1469         c = changes[0]
1470         if c['User'] == newUser: return   # nothing to do
1471         c['User'] = newUser
1472         input = marshal.dumps(c)
1473
1474         result = p4CmdList("change -f -i", stdin=input)
1475         for r in result:
1476             if r.has_key('code'):
1477                 if r['code'] == 'error':
1478                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1479             if r.has_key('data'):
1480                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1481                 return
1482         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1483
1484     def canChangeChangelists(self):
1485         # check to see if we have p4 admin or super-user permissions, either of
1486         # which are required to modify changelists.
1487         results = p4CmdList(["protects", self.depotPath])
1488         for r in results:
1489             if r.has_key('perm'):
1490                 if r['perm'] == 'admin':
1491                     return 1
1492                 if r['perm'] == 'super':
1493                     return 1
1494         return 0
1495
1496     def prepareSubmitTemplate(self, changelist=None):
1497         """Run "p4 change -o" to grab a change specification template.
1498            This does not use "p4 -G", as it is nice to keep the submission
1499            template in original order, since a human might edit it.
1500
1501            Remove lines in the Files section that show changes to files
1502            outside the depot path we're committing into."""
1503
1504         [upstream, settings] = findUpstreamBranchPoint()
1505
1506         template = ""
1507         inFilesSection = False
1508         args = ['change', '-o']
1509         if changelist:
1510             args.append(str(changelist))
1511
1512         for line in p4_read_pipe_lines(args):
1513             if line.endswith("\r\n"):
1514                 line = line[:-2] + "\n"
1515             if inFilesSection:
1516                 if line.startswith("\t"):
1517                     # path starts and ends with a tab
1518                     path = line[1:]
1519                     lastTab = path.rfind("\t")
1520                     if lastTab != -1:
1521                         path = path[:lastTab]
1522                         if settings.has_key('depot-paths'):
1523                             if not [p for p in settings['depot-paths']
1524                                     if p4PathStartsWith(path, p)]:
1525                                 continue
1526                         else:
1527                             if not p4PathStartsWith(path, self.depotPath):
1528                                 continue
1529                 else:
1530                     inFilesSection = False
1531             else:
1532                 if line.startswith("Files:"):
1533                     inFilesSection = True
1534
1535             template += line
1536
1537         return template
1538
1539     def edit_template(self, template_file):
1540         """Invoke the editor to let the user change the submission
1541            message.  Return true if okay to continue with the submit."""
1542
1543         # if configured to skip the editing part, just submit
1544         if gitConfigBool("git-p4.skipSubmitEdit"):
1545             return True
1546
1547         # look at the modification time, to check later if the user saved
1548         # the file
1549         mtime = os.stat(template_file).st_mtime
1550
1551         # invoke the editor
1552         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1553             editor = os.environ.get("P4EDITOR")
1554         else:
1555             editor = read_pipe("git var GIT_EDITOR").strip()
1556         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1557
1558         # If the file was not saved, prompt to see if this patch should
1559         # be skipped.  But skip this verification step if configured so.
1560         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1561             return True
1562
1563         # modification time updated means user saved the file
1564         if os.stat(template_file).st_mtime > mtime:
1565             return True
1566
1567         while True:
1568             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1569             if response == 'y':
1570                 return True
1571             if response == 'n':
1572                 return False
1573
1574     def get_diff_description(self, editedFiles, filesToAdd):
1575         # diff
1576         if os.environ.has_key("P4DIFF"):
1577             del(os.environ["P4DIFF"])
1578         diff = ""
1579         for editedFile in editedFiles:
1580             diff += p4_read_pipe(['diff', '-du',
1581                                   wildcard_encode(editedFile)])
1582
1583         # new file diff
1584         newdiff = ""
1585         for newFile in filesToAdd:
1586             newdiff += "==== new file ====\n"
1587             newdiff += "--- /dev/null\n"
1588             newdiff += "+++ %s\n" % newFile
1589             f = open(newFile, "r")
1590             for line in f.readlines():
1591                 newdiff += "+" + line
1592             f.close()
1593
1594         return (diff + newdiff).replace('\r\n', '\n')
1595
1596     def applyCommit(self, id):
1597         """Apply one commit, return True if it succeeded."""
1598
1599         print "Applying", read_pipe(["git", "show", "-s",
1600                                      "--format=format:%h %s", id])
1601
1602         (p4User, gitEmail) = self.p4UserForCommit(id)
1603
1604         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1605         filesToAdd = set()
1606         filesToChangeType = set()
1607         filesToDelete = set()
1608         editedFiles = set()
1609         pureRenameCopy = set()
1610         filesToChangeExecBit = {}
1611         all_files = list()
1612
1613         for line in diff:
1614             diff = parseDiffTreeEntry(line)
1615             modifier = diff['status']
1616             path = diff['src']
1617             all_files.append(path)
1618
1619             if modifier == "M":
1620                 p4_edit(path)
1621                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1622                     filesToChangeExecBit[path] = diff['dst_mode']
1623                 editedFiles.add(path)
1624             elif modifier == "A":
1625                 filesToAdd.add(path)
1626                 filesToChangeExecBit[path] = diff['dst_mode']
1627                 if path in filesToDelete:
1628                     filesToDelete.remove(path)
1629             elif modifier == "D":
1630                 filesToDelete.add(path)
1631                 if path in filesToAdd:
1632                     filesToAdd.remove(path)
1633             elif modifier == "C":
1634                 src, dest = diff['src'], diff['dst']
1635                 p4_integrate(src, dest)
1636                 pureRenameCopy.add(dest)
1637                 if diff['src_sha1'] != diff['dst_sha1']:
1638                     p4_edit(dest)
1639                     pureRenameCopy.discard(dest)
1640                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1641                     p4_edit(dest)
1642                     pureRenameCopy.discard(dest)
1643                     filesToChangeExecBit[dest] = diff['dst_mode']
1644                 if self.isWindows:
1645                     # turn off read-only attribute
1646                     os.chmod(dest, stat.S_IWRITE)
1647                 os.unlink(dest)
1648                 editedFiles.add(dest)
1649             elif modifier == "R":
1650                 src, dest = diff['src'], diff['dst']
1651                 if self.p4HasMoveCommand:
1652                     p4_edit(src)        # src must be open before move
1653                     p4_move(src, dest)  # opens for (move/delete, move/add)
1654                 else:
1655                     p4_integrate(src, dest)
1656                     if diff['src_sha1'] != diff['dst_sha1']:
1657                         p4_edit(dest)
1658                     else:
1659                         pureRenameCopy.add(dest)
1660                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1661                     if not self.p4HasMoveCommand:
1662                         p4_edit(dest)   # with move: already open, writable
1663                     filesToChangeExecBit[dest] = diff['dst_mode']
1664                 if not self.p4HasMoveCommand:
1665                     if self.isWindows:
1666                         os.chmod(dest, stat.S_IWRITE)
1667                     os.unlink(dest)
1668                     filesToDelete.add(src)
1669                 editedFiles.add(dest)
1670             elif modifier == "T":
1671                 filesToChangeType.add(path)
1672             else:
1673                 die("unknown modifier %s for %s" % (modifier, path))
1674
1675         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1676         patchcmd = diffcmd + " | git apply "
1677         tryPatchCmd = patchcmd + "--check -"
1678         applyPatchCmd = patchcmd + "--check --apply -"
1679         patch_succeeded = True
1680
1681         if os.system(tryPatchCmd) != 0:
1682             fixed_rcs_keywords = False
1683             patch_succeeded = False
1684             print "Unfortunately applying the change failed!"
1685
1686             # Patch failed, maybe it's just RCS keyword woes. Look through
1687             # the patch to see if that's possible.
1688             if gitConfigBool("git-p4.attemptRCSCleanup"):
1689                 file = None
1690                 pattern = None
1691                 kwfiles = {}
1692                 for file in editedFiles | filesToDelete:
1693                     # did this file's delta contain RCS keywords?
1694                     pattern = p4_keywords_regexp_for_file(file)
1695
1696                     if pattern:
1697                         # this file is a possibility...look for RCS keywords.
1698                         regexp = re.compile(pattern, re.VERBOSE)
1699                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1700                             if regexp.search(line):
1701                                 if verbose:
1702                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1703                                 kwfiles[file] = pattern
1704                                 break
1705
1706                 for file in kwfiles:
1707                     if verbose:
1708                         print "zapping %s with %s" % (line,pattern)
1709                     # File is being deleted, so not open in p4.  Must
1710                     # disable the read-only bit on windows.
1711                     if self.isWindows and file not in editedFiles:
1712                         os.chmod(file, stat.S_IWRITE)
1713                     self.patchRCSKeywords(file, kwfiles[file])
1714                     fixed_rcs_keywords = True
1715
1716             if fixed_rcs_keywords:
1717                 print "Retrying the patch with RCS keywords cleaned up"
1718                 if os.system(tryPatchCmd) == 0:
1719                     patch_succeeded = True
1720
1721         if not patch_succeeded:
1722             for f in editedFiles:
1723                 p4_revert(f)
1724             return False
1725
1726         #
1727         # Apply the patch for real, and do add/delete/+x handling.
1728         #
1729         system(applyPatchCmd)
1730
1731         for f in filesToChangeType:
1732             p4_edit(f, "-t", "auto")
1733         for f in filesToAdd:
1734             p4_add(f)
1735         for f in filesToDelete:
1736             p4_revert(f)
1737             p4_delete(f)
1738
1739         # Set/clear executable bits
1740         for f in filesToChangeExecBit.keys():
1741             mode = filesToChangeExecBit[f]
1742             setP4ExecBit(f, mode)
1743
1744         if self.update_shelve:
1745             print("all_files = %s" % str(all_files))
1746             p4_reopen_in_change(self.update_shelve, all_files)
1747
1748         #
1749         # Build p4 change description, starting with the contents
1750         # of the git commit message.
1751         #
1752         logMessage = extractLogMessageFromGitCommit(id)
1753         logMessage = logMessage.strip()
1754         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1755
1756         template = self.prepareSubmitTemplate(self.update_shelve)
1757         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1758
1759         if self.preserveUser:
1760            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1761
1762         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1763             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1764             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1765             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1766
1767         separatorLine = "######## everything below this line is just the diff #######\n"
1768         if not self.prepare_p4_only:
1769             submitTemplate += separatorLine
1770             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1771
1772         (handle, fileName) = tempfile.mkstemp()
1773         tmpFile = os.fdopen(handle, "w+b")
1774         if self.isWindows:
1775             submitTemplate = submitTemplate.replace("\n", "\r\n")
1776         tmpFile.write(submitTemplate)
1777         tmpFile.close()
1778
1779         if self.prepare_p4_only:
1780             #
1781             # Leave the p4 tree prepared, and the submit template around
1782             # and let the user decide what to do next
1783             #
1784             print
1785             print "P4 workspace prepared for submission."
1786             print "To submit or revert, go to client workspace"
1787             print "  " + self.clientPath
1788             print
1789             print "To submit, use \"p4 submit\" to write a new description,"
1790             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1791                   " \"git p4\"." % fileName
1792             print "You can delete the file \"%s\" when finished." % fileName
1793
1794             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1795                 print "To preserve change ownership by user %s, you must\n" \
1796                       "do \"p4 change -f <change>\" after submitting and\n" \
1797                       "edit the User field."
1798             if pureRenameCopy:
1799                 print "After submitting, renamed files must be re-synced."
1800                 print "Invoke \"p4 sync -f\" on each of these files:"
1801                 for f in pureRenameCopy:
1802                     print "  " + f
1803
1804             print
1805             print "To revert the changes, use \"p4 revert ...\", and delete"
1806             print "the submit template file \"%s\"" % fileName
1807             if filesToAdd:
1808                 print "Since the commit adds new files, they must be deleted:"
1809                 for f in filesToAdd:
1810                     print "  " + f
1811             print
1812             return True
1813
1814         #
1815         # Let the user edit the change description, then submit it.
1816         #
1817         submitted = False
1818
1819         try:
1820             if self.edit_template(fileName):
1821                 # read the edited message and submit
1822                 tmpFile = open(fileName, "rb")
1823                 message = tmpFile.read()
1824                 tmpFile.close()
1825                 if self.isWindows:
1826                     message = message.replace("\r\n", "\n")
1827                 submitTemplate = message[:message.index(separatorLine)]
1828
1829                 if self.update_shelve:
1830                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1831                 elif self.shelve:
1832                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1833                 else:
1834                     p4_write_pipe(['submit', '-i'], submitTemplate)
1835                     # The rename/copy happened by applying a patch that created a
1836                     # new file.  This leaves it writable, which confuses p4.
1837                     for f in pureRenameCopy:
1838                         p4_sync(f, "-f")
1839
1840                 if self.preserveUser:
1841                     if p4User:
1842                         # Get last changelist number. Cannot easily get it from
1843                         # the submit command output as the output is
1844                         # unmarshalled.
1845                         changelist = self.lastP4Changelist()
1846                         self.modifyChangelistUser(changelist, p4User)
1847
1848                 submitted = True
1849
1850         finally:
1851             # skip this patch
1852             if not submitted or self.shelve:
1853                 if self.shelve:
1854                     print ("Reverting shelved files.")
1855                 else:
1856                     print ("Submission cancelled, undoing p4 changes.")
1857                 for f in editedFiles | filesToDelete:
1858                     p4_revert(f)
1859                 for f in filesToAdd:
1860                     p4_revert(f)
1861                     os.remove(f)
1862
1863         os.remove(fileName)
1864         return submitted
1865
1866     # Export git tags as p4 labels. Create a p4 label and then tag
1867     # with that.
1868     def exportGitTags(self, gitTags):
1869         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1870         if len(validLabelRegexp) == 0:
1871             validLabelRegexp = defaultLabelRegexp
1872         m = re.compile(validLabelRegexp)
1873
1874         for name in gitTags:
1875
1876             if not m.match(name):
1877                 if verbose:
1878                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1879                 continue
1880
1881             # Get the p4 commit this corresponds to
1882             logMessage = extractLogMessageFromGitCommit(name)
1883             values = extractSettingsGitLog(logMessage)
1884
1885             if not values.has_key('change'):
1886                 # a tag pointing to something not sent to p4; ignore
1887                 if verbose:
1888                     print "git tag %s does not give a p4 commit" % name
1889                 continue
1890             else:
1891                 changelist = values['change']
1892
1893             # Get the tag details.
1894             inHeader = True
1895             isAnnotated = False
1896             body = []
1897             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1898                 l = l.strip()
1899                 if inHeader:
1900                     if re.match(r'tag\s+', l):
1901                         isAnnotated = True
1902                     elif re.match(r'\s*$', l):
1903                         inHeader = False
1904                         continue
1905                 else:
1906                     body.append(l)
1907
1908             if not isAnnotated:
1909                 body = ["lightweight tag imported by git p4\n"]
1910
1911             # Create the label - use the same view as the client spec we are using
1912             clientSpec = getClientSpec()
1913
1914             labelTemplate  = "Label: %s\n" % name
1915             labelTemplate += "Description:\n"
1916             for b in body:
1917                 labelTemplate += "\t" + b + "\n"
1918             labelTemplate += "View:\n"
1919             for depot_side in clientSpec.mappings:
1920                 labelTemplate += "\t%s\n" % depot_side
1921
1922             if self.dry_run:
1923                 print "Would create p4 label %s for tag" % name
1924             elif self.prepare_p4_only:
1925                 print "Not creating p4 label %s for tag due to option" \
1926                       " --prepare-p4-only" % name
1927             else:
1928                 p4_write_pipe(["label", "-i"], labelTemplate)
1929
1930                 # Use the label
1931                 p4_system(["tag", "-l", name] +
1932                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1933
1934                 if verbose:
1935                     print "created p4 label for tag %s" % name
1936
1937     def run(self, args):
1938         if len(args) == 0:
1939             self.master = currentGitBranch()
1940         elif len(args) == 1:
1941             self.master = args[0]
1942             if not branchExists(self.master):
1943                 die("Branch %s does not exist" % self.master)
1944         else:
1945             return False
1946
1947         if self.master:
1948             allowSubmit = gitConfig("git-p4.allowSubmit")
1949             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1950                 die("%s is not in git-p4.allowSubmit" % self.master)
1951
1952         [upstream, settings] = findUpstreamBranchPoint()
1953         self.depotPath = settings['depot-paths'][0]
1954         if len(self.origin) == 0:
1955             self.origin = upstream
1956
1957         if self.update_shelve:
1958             self.shelve = True
1959
1960         if self.preserveUser:
1961             if not self.canChangeChangelists():
1962                 die("Cannot preserve user names without p4 super-user or admin permissions")
1963
1964         # if not set from the command line, try the config file
1965         if self.conflict_behavior is None:
1966             val = gitConfig("git-p4.conflict")
1967             if val:
1968                 if val not in self.conflict_behavior_choices:
1969                     die("Invalid value '%s' for config git-p4.conflict" % val)
1970             else:
1971                 val = "ask"
1972             self.conflict_behavior = val
1973
1974         if self.verbose:
1975             print "Origin branch is " + self.origin
1976
1977         if len(self.depotPath) == 0:
1978             print "Internal error: cannot locate perforce depot path from existing branches"
1979             sys.exit(128)
1980
1981         self.useClientSpec = False
1982         if gitConfigBool("git-p4.useclientspec"):
1983             self.useClientSpec = True
1984         if self.useClientSpec:
1985             self.clientSpecDirs = getClientSpec()
1986
1987         # Check for the existence of P4 branches
1988         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1989
1990         if self.useClientSpec and not branchesDetected:
1991             # all files are relative to the client spec
1992             self.clientPath = getClientRoot()
1993         else:
1994             self.clientPath = p4Where(self.depotPath)
1995
1996         if self.clientPath == "":
1997             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1998
1999         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2000         self.oldWorkingDirectory = os.getcwd()
2001
2002         # ensure the clientPath exists
2003         new_client_dir = False
2004         if not os.path.exists(self.clientPath):
2005             new_client_dir = True
2006             os.makedirs(self.clientPath)
2007
2008         chdir(self.clientPath, is_client_path=True)
2009         if self.dry_run:
2010             print "Would synchronize p4 checkout in %s" % self.clientPath
2011         else:
2012             print "Synchronizing p4 checkout..."
2013             if new_client_dir:
2014                 # old one was destroyed, and maybe nobody told p4
2015                 p4_sync("...", "-f")
2016             else:
2017                 p4_sync("...")
2018         self.check()
2019
2020         commits = []
2021         if self.master:
2022             commitish = self.master
2023         else:
2024             commitish = 'HEAD'
2025
2026         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2027             commits.append(line.strip())
2028         commits.reverse()
2029
2030         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2031             self.checkAuthorship = False
2032         else:
2033             self.checkAuthorship = True
2034
2035         if self.preserveUser:
2036             self.checkValidP4Users(commits)
2037
2038         #
2039         # Build up a set of options to be passed to diff when
2040         # submitting each commit to p4.
2041         #
2042         if self.detectRenames:
2043             # command-line -M arg
2044             self.diffOpts = "-M"
2045         else:
2046             # If not explicitly set check the config variable
2047             detectRenames = gitConfig("git-p4.detectRenames")
2048
2049             if detectRenames.lower() == "false" or detectRenames == "":
2050                 self.diffOpts = ""
2051             elif detectRenames.lower() == "true":
2052                 self.diffOpts = "-M"
2053             else:
2054                 self.diffOpts = "-M%s" % detectRenames
2055
2056         # no command-line arg for -C or --find-copies-harder, just
2057         # config variables
2058         detectCopies = gitConfig("git-p4.detectCopies")
2059         if detectCopies.lower() == "false" or detectCopies == "":
2060             pass
2061         elif detectCopies.lower() == "true":
2062             self.diffOpts += " -C"
2063         else:
2064             self.diffOpts += " -C%s" % detectCopies
2065
2066         if gitConfigBool("git-p4.detectCopiesHarder"):
2067             self.diffOpts += " --find-copies-harder"
2068
2069         #
2070         # Apply the commits, one at a time.  On failure, ask if should
2071         # continue to try the rest of the patches, or quit.
2072         #
2073         if self.dry_run:
2074             print "Would apply"
2075         applied = []
2076         last = len(commits) - 1
2077         for i, commit in enumerate(commits):
2078             if self.dry_run:
2079                 print " ", read_pipe(["git", "show", "-s",
2080                                       "--format=format:%h %s", commit])
2081                 ok = True
2082             else:
2083                 ok = self.applyCommit(commit)
2084             if ok:
2085                 applied.append(commit)
2086             else:
2087                 if self.prepare_p4_only and i < last:
2088                     print "Processing only the first commit due to option" \
2089                           " --prepare-p4-only"
2090                     break
2091                 if i < last:
2092                     quit = False
2093                     while True:
2094                         # prompt for what to do, or use the option/variable
2095                         if self.conflict_behavior == "ask":
2096                             print "What do you want to do?"
2097                             response = raw_input("[s]kip this commit but apply"
2098                                                  " the rest, or [q]uit? ")
2099                             if not response:
2100                                 continue
2101                         elif self.conflict_behavior == "skip":
2102                             response = "s"
2103                         elif self.conflict_behavior == "quit":
2104                             response = "q"
2105                         else:
2106                             die("Unknown conflict_behavior '%s'" %
2107                                 self.conflict_behavior)
2108
2109                         if response[0] == "s":
2110                             print "Skipping this commit, but applying the rest"
2111                             break
2112                         if response[0] == "q":
2113                             print "Quitting"
2114                             quit = True
2115                             break
2116                     if quit:
2117                         break
2118
2119         chdir(self.oldWorkingDirectory)
2120         shelved_applied = "shelved" if self.shelve else "applied"
2121         if self.dry_run:
2122             pass
2123         elif self.prepare_p4_only:
2124             pass
2125         elif len(commits) == len(applied):
2126             print ("All commits {0}!".format(shelved_applied))
2127
2128             sync = P4Sync()
2129             if self.branch:
2130                 sync.branch = self.branch
2131             sync.run([])
2132
2133             rebase = P4Rebase()
2134             rebase.rebase()
2135
2136         else:
2137             if len(applied) == 0:
2138                 print ("No commits {0}.".format(shelved_applied))
2139             else:
2140                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2141                 for c in commits:
2142                     if c in applied:
2143                         star = "*"
2144                     else:
2145                         star = " "
2146                     print star, read_pipe(["git", "show", "-s",
2147                                            "--format=format:%h %s",  c])
2148                 print "You will have to do 'git p4 sync' and rebase."
2149
2150         if gitConfigBool("git-p4.exportLabels"):
2151             self.exportLabels = True
2152
2153         if self.exportLabels:
2154             p4Labels = getP4Labels(self.depotPath)
2155             gitTags = getGitTags()
2156
2157             missingGitTags = gitTags - p4Labels
2158             self.exportGitTags(missingGitTags)
2159
2160         # exit with error unless everything applied perfectly
2161         if len(commits) != len(applied):
2162                 sys.exit(1)
2163
2164         return True
2165
2166 class View(object):
2167     """Represent a p4 view ("p4 help views"), and map files in a
2168        repo according to the view."""
2169
2170     def __init__(self, client_name):
2171         self.mappings = []
2172         self.client_prefix = "//%s/" % client_name
2173         # cache results of "p4 where" to lookup client file locations
2174         self.client_spec_path_cache = {}
2175
2176     def append(self, view_line):
2177         """Parse a view line, splitting it into depot and client
2178            sides.  Append to self.mappings, preserving order.  This
2179            is only needed for tag creation."""
2180
2181         # Split the view line into exactly two words.  P4 enforces
2182         # structure on these lines that simplifies this quite a bit.
2183         #
2184         # Either or both words may be double-quoted.
2185         # Single quotes do not matter.
2186         # Double-quote marks cannot occur inside the words.
2187         # A + or - prefix is also inside the quotes.
2188         # There are no quotes unless they contain a space.
2189         # The line is already white-space stripped.
2190         # The two words are separated by a single space.
2191         #
2192         if view_line[0] == '"':
2193             # First word is double quoted.  Find its end.
2194             close_quote_index = view_line.find('"', 1)
2195             if close_quote_index <= 0:
2196                 die("No first-word closing quote found: %s" % view_line)
2197             depot_side = view_line[1:close_quote_index]
2198             # skip closing quote and space
2199             rhs_index = close_quote_index + 1 + 1
2200         else:
2201             space_index = view_line.find(" ")
2202             if space_index <= 0:
2203                 die("No word-splitting space found: %s" % view_line)
2204             depot_side = view_line[0:space_index]
2205             rhs_index = space_index + 1
2206
2207         # prefix + means overlay on previous mapping
2208         if depot_side.startswith("+"):
2209             depot_side = depot_side[1:]
2210
2211         # prefix - means exclude this path, leave out of mappings
2212         exclude = False
2213         if depot_side.startswith("-"):
2214             exclude = True
2215             depot_side = depot_side[1:]
2216
2217         if not exclude:
2218             self.mappings.append(depot_side)
2219
2220     def convert_client_path(self, clientFile):
2221         # chop off //client/ part to make it relative
2222         if not clientFile.startswith(self.client_prefix):
2223             die("No prefix '%s' on clientFile '%s'" %
2224                 (self.client_prefix, clientFile))
2225         return clientFile[len(self.client_prefix):]
2226
2227     def update_client_spec_path_cache(self, files):
2228         """ Caching file paths by "p4 where" batch query """
2229
2230         # List depot file paths exclude that already cached
2231         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2232
2233         if len(fileArgs) == 0:
2234             return  # All files in cache
2235
2236         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2237         for res in where_result:
2238             if "code" in res and res["code"] == "error":
2239                 # assume error is "... file(s) not in client view"
2240                 continue
2241             if "clientFile" not in res:
2242                 die("No clientFile in 'p4 where' output")
2243             if "unmap" in res:
2244                 # it will list all of them, but only one not unmap-ped
2245                 continue
2246             if gitConfigBool("core.ignorecase"):
2247                 res['depotFile'] = res['depotFile'].lower()
2248             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2249
2250         # not found files or unmap files set to ""
2251         for depotFile in fileArgs:
2252             if gitConfigBool("core.ignorecase"):
2253                 depotFile = depotFile.lower()
2254             if depotFile not in self.client_spec_path_cache:
2255                 self.client_spec_path_cache[depotFile] = ""
2256
2257     def map_in_client(self, depot_path):
2258         """Return the relative location in the client where this
2259            depot file should live.  Returns "" if the file should
2260            not be mapped in the client."""
2261
2262         if gitConfigBool("core.ignorecase"):
2263             depot_path = depot_path.lower()
2264
2265         if depot_path in self.client_spec_path_cache:
2266             return self.client_spec_path_cache[depot_path]
2267
2268         die( "Error: %s is not found in client spec path" % depot_path )
2269         return ""
2270
2271 class P4Sync(Command, P4UserMap):
2272     delete_actions = ( "delete", "move/delete", "purge" )
2273
2274     def __init__(self):
2275         Command.__init__(self)
2276         P4UserMap.__init__(self)
2277         self.options = [
2278                 optparse.make_option("--branch", dest="branch"),
2279                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2280                 optparse.make_option("--changesfile", dest="changesFile"),
2281                 optparse.make_option("--silent", dest="silent", action="store_true"),
2282                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2283                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2284                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2285                                      help="Import into refs/heads/ , not refs/remotes"),
2286                 optparse.make_option("--max-changes", dest="maxChanges",
2287                                      help="Maximum number of changes to import"),
2288                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2289                                      help="Internal block size to use when iteratively calling p4 changes"),
2290                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2291                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2292                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2293                                      help="Only sync files that are included in the Perforce Client Spec"),
2294                 optparse.make_option("-/", dest="cloneExclude",
2295                                      action="append", type="string",
2296                                      help="exclude depot path"),
2297         ]
2298         self.description = """Imports from Perforce into a git repository.\n
2299     example:
2300     //depot/my/project/ -- to import the current head
2301     //depot/my/project/@all -- to import everything
2302     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2303
2304     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2305
2306         self.usage += " //depot/path[@revRange]"
2307         self.silent = False
2308         self.createdBranches = set()
2309         self.committedChanges = set()
2310         self.branch = ""
2311         self.detectBranches = False
2312         self.detectLabels = False
2313         self.importLabels = False
2314         self.changesFile = ""
2315         self.syncWithOrigin = True
2316         self.importIntoRemotes = True
2317         self.maxChanges = ""
2318         self.changes_block_size = None
2319         self.keepRepoPath = False
2320         self.depotPaths = None
2321         self.p4BranchesInGit = []
2322         self.cloneExclude = []
2323         self.useClientSpec = False
2324         self.useClientSpec_from_options = False
2325         self.clientSpecDirs = None
2326         self.tempBranches = []
2327         self.tempBranchLocation = "refs/git-p4-tmp"
2328         self.largeFileSystem = None
2329
2330         if gitConfig('git-p4.largeFileSystem'):
2331             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2332             self.largeFileSystem = largeFileSystemConstructor(
2333                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2334             )
2335
2336         if gitConfig("git-p4.syncFromOrigin") == "false":
2337             self.syncWithOrigin = False
2338
2339     # This is required for the "append" cloneExclude action
2340     def ensure_value(self, attr, value):
2341         if not hasattr(self, attr) or getattr(self, attr) is None:
2342             setattr(self, attr, value)
2343         return getattr(self, attr)
2344
2345     # Force a checkpoint in fast-import and wait for it to finish
2346     def checkpoint(self):
2347         self.gitStream.write("checkpoint\n\n")
2348         self.gitStream.write("progress checkpoint\n\n")
2349         out = self.gitOutput.readline()
2350         if self.verbose:
2351             print "checkpoint finished: " + out
2352
2353     def extractFilesFromCommit(self, commit):
2354         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2355                              for path in self.cloneExclude]
2356         files = []
2357         fnum = 0
2358         while commit.has_key("depotFile%s" % fnum):
2359             path =  commit["depotFile%s" % fnum]
2360
2361             if [p for p in self.cloneExclude
2362                 if p4PathStartsWith(path, p)]:
2363                 found = False
2364             else:
2365                 found = [p for p in self.depotPaths
2366                          if p4PathStartsWith(path, p)]
2367             if not found:
2368                 fnum = fnum + 1
2369                 continue
2370
2371             file = {}
2372             file["path"] = path
2373             file["rev"] = commit["rev%s" % fnum]
2374             file["action"] = commit["action%s" % fnum]
2375             file["type"] = commit["type%s" % fnum]
2376             files.append(file)
2377             fnum = fnum + 1
2378         return files
2379
2380     def extractJobsFromCommit(self, commit):
2381         jobs = []
2382         jnum = 0
2383         while commit.has_key("job%s" % jnum):
2384             job = commit["job%s" % jnum]
2385             jobs.append(job)
2386             jnum = jnum + 1
2387         return jobs
2388
2389     def stripRepoPath(self, path, prefixes):
2390         """When streaming files, this is called to map a p4 depot path
2391            to where it should go in git.  The prefixes are either
2392            self.depotPaths, or self.branchPrefixes in the case of
2393            branch detection."""
2394
2395         if self.useClientSpec:
2396             # branch detection moves files up a level (the branch name)
2397             # from what client spec interpretation gives
2398             path = self.clientSpecDirs.map_in_client(path)
2399             if self.detectBranches:
2400                 for b in self.knownBranches:
2401                     if path.startswith(b + "/"):
2402                         path = path[len(b)+1:]
2403
2404         elif self.keepRepoPath:
2405             # Preserve everything in relative path name except leading
2406             # //depot/; just look at first prefix as they all should
2407             # be in the same depot.
2408             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2409             if p4PathStartsWith(path, depot):
2410                 path = path[len(depot):]
2411
2412         else:
2413             for p in prefixes:
2414                 if p4PathStartsWith(path, p):
2415                     path = path[len(p):]
2416                     break
2417
2418         path = wildcard_decode(path)
2419         return path
2420
2421     def splitFilesIntoBranches(self, commit):
2422         """Look at each depotFile in the commit to figure out to what
2423            branch it belongs."""
2424
2425         if self.clientSpecDirs:
2426             files = self.extractFilesFromCommit(commit)
2427             self.clientSpecDirs.update_client_spec_path_cache(files)
2428
2429         branches = {}
2430         fnum = 0
2431         while commit.has_key("depotFile%s" % fnum):
2432             path =  commit["depotFile%s" % fnum]
2433             found = [p for p in self.depotPaths
2434                      if p4PathStartsWith(path, p)]
2435             if not found:
2436                 fnum = fnum + 1
2437                 continue
2438
2439             file = {}
2440             file["path"] = path
2441             file["rev"] = commit["rev%s" % fnum]
2442             file["action"] = commit["action%s" % fnum]
2443             file["type"] = commit["type%s" % fnum]
2444             fnum = fnum + 1
2445
2446             # start with the full relative path where this file would
2447             # go in a p4 client
2448             if self.useClientSpec:
2449                 relPath = self.clientSpecDirs.map_in_client(path)
2450             else:
2451                 relPath = self.stripRepoPath(path, self.depotPaths)
2452
2453             for branch in self.knownBranches.keys():
2454                 # add a trailing slash so that a commit into qt/4.2foo
2455                 # doesn't end up in qt/4.2, e.g.
2456                 if relPath.startswith(branch + "/"):
2457                     if branch not in branches:
2458                         branches[branch] = []
2459                     branches[branch].append(file)
2460                     break
2461
2462         return branches
2463
2464     def writeToGitStream(self, gitMode, relPath, contents):
2465         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2466         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2467         for d in contents:
2468             self.gitStream.write(d)
2469         self.gitStream.write('\n')
2470
2471     # output one file from the P4 stream
2472     # - helper for streamP4Files
2473
2474     def streamOneP4File(self, file, contents):
2475         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2476         if verbose:
2477             size = int(self.stream_file['fileSize'])
2478             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2479             sys.stdout.flush()
2480
2481         (type_base, type_mods) = split_p4_type(file["type"])
2482
2483         git_mode = "100644"
2484         if "x" in type_mods:
2485             git_mode = "100755"
2486         if type_base == "symlink":
2487             git_mode = "120000"
2488             # p4 print on a symlink sometimes contains "target\n";
2489             # if it does, remove the newline
2490             data = ''.join(contents)
2491             if not data:
2492                 # Some version of p4 allowed creating a symlink that pointed
2493                 # to nothing.  This causes p4 errors when checking out such
2494                 # a change, and errors here too.  Work around it by ignoring
2495                 # the bad symlink; hopefully a future change fixes it.
2496                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2497                 return
2498             elif data[-1] == '\n':
2499                 contents = [data[:-1]]
2500             else:
2501                 contents = [data]
2502
2503         if type_base == "utf16":
2504             # p4 delivers different text in the python output to -G
2505             # than it does when using "print -o", or normal p4 client
2506             # operations.  utf16 is converted to ascii or utf8, perhaps.
2507             # But ascii text saved as -t utf16 is completely mangled.
2508             # Invoke print -o to get the real contents.
2509             #
2510             # On windows, the newlines will always be mangled by print, so put
2511             # them back too.  This is not needed to the cygwin windows version,
2512             # just the native "NT" type.
2513             #
2514             try:
2515                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2516             except Exception as e:
2517                 if 'Translation of file content failed' in str(e):
2518                     type_base = 'binary'
2519                 else:
2520                     raise e
2521             else:
2522                 if p4_version_string().find('/NT') >= 0:
2523                     text = text.replace('\r\n', '\n')
2524                 contents = [ text ]
2525
2526         if type_base == "apple":
2527             # Apple filetype files will be streamed as a concatenation of
2528             # its appledouble header and the contents.  This is useless
2529             # on both macs and non-macs.  If using "print -q -o xx", it
2530             # will create "xx" with the data, and "%xx" with the header.
2531             # This is also not very useful.
2532             #
2533             # Ideally, someday, this script can learn how to generate
2534             # appledouble files directly and import those to git, but
2535             # non-mac machines can never find a use for apple filetype.
2536             print "\nIgnoring apple filetype file %s" % file['depotFile']
2537             return
2538
2539         # Note that we do not try to de-mangle keywords on utf16 files,
2540         # even though in theory somebody may want that.
2541         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2542         if pattern:
2543             regexp = re.compile(pattern, re.VERBOSE)
2544             text = ''.join(contents)
2545             text = regexp.sub(r'$\1$', text)
2546             contents = [ text ]
2547
2548         try:
2549             relPath.decode('ascii')
2550         except:
2551             encoding = 'utf8'
2552             if gitConfig('git-p4.pathEncoding'):
2553                 encoding = gitConfig('git-p4.pathEncoding')
2554             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2555             if self.verbose:
2556                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2557
2558         if self.largeFileSystem:
2559             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2560
2561         self.writeToGitStream(git_mode, relPath, contents)
2562
2563     def streamOneP4Deletion(self, file):
2564         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2565         if verbose:
2566             sys.stdout.write("delete %s\n" % relPath)
2567             sys.stdout.flush()
2568         self.gitStream.write("D %s\n" % relPath)
2569
2570         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2571             self.largeFileSystem.removeLargeFile(relPath)
2572
2573     # handle another chunk of streaming data
2574     def streamP4FilesCb(self, marshalled):
2575
2576         # catch p4 errors and complain
2577         err = None
2578         if "code" in marshalled:
2579             if marshalled["code"] == "error":
2580                 if "data" in marshalled:
2581                     err = marshalled["data"].rstrip()
2582
2583         if not err and 'fileSize' in self.stream_file:
2584             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2585             if required_bytes > 0:
2586                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2587                     os.getcwd(), required_bytes/1024/1024
2588                 )
2589
2590         if err:
2591             f = None
2592             if self.stream_have_file_info:
2593                 if "depotFile" in self.stream_file:
2594                     f = self.stream_file["depotFile"]
2595             # force a failure in fast-import, else an empty
2596             # commit will be made
2597             self.gitStream.write("\n")
2598             self.gitStream.write("die-now\n")
2599             self.gitStream.close()
2600             # ignore errors, but make sure it exits first
2601             self.importProcess.wait()
2602             if f:
2603                 die("Error from p4 print for %s: %s" % (f, err))
2604             else:
2605                 die("Error from p4 print: %s" % err)
2606
2607         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2608             # start of a new file - output the old one first
2609             self.streamOneP4File(self.stream_file, self.stream_contents)
2610             self.stream_file = {}
2611             self.stream_contents = []
2612             self.stream_have_file_info = False
2613
2614         # pick up the new file information... for the
2615         # 'data' field we need to append to our array
2616         for k in marshalled.keys():
2617             if k == 'data':
2618                 if 'streamContentSize' not in self.stream_file:
2619                     self.stream_file['streamContentSize'] = 0
2620                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2621                 self.stream_contents.append(marshalled['data'])
2622             else:
2623                 self.stream_file[k] = marshalled[k]
2624
2625         if (verbose and
2626             'streamContentSize' in self.stream_file and
2627             'fileSize' in self.stream_file and
2628             'depotFile' in self.stream_file):
2629             size = int(self.stream_file["fileSize"])
2630             if size > 0:
2631                 progress = 100*self.stream_file['streamContentSize']/size
2632                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2633                 sys.stdout.flush()
2634
2635         self.stream_have_file_info = True
2636
2637     # Stream directly from "p4 files" into "git fast-import"
2638     def streamP4Files(self, files):
2639         filesForCommit = []
2640         filesToRead = []
2641         filesToDelete = []
2642
2643         for f in files:
2644             filesForCommit.append(f)
2645             if f['action'] in self.delete_actions:
2646                 filesToDelete.append(f)
2647             else:
2648                 filesToRead.append(f)
2649
2650         # deleted files...
2651         for f in filesToDelete:
2652             self.streamOneP4Deletion(f)
2653
2654         if len(filesToRead) > 0:
2655             self.stream_file = {}
2656             self.stream_contents = []
2657             self.stream_have_file_info = False
2658
2659             # curry self argument
2660             def streamP4FilesCbSelf(entry):
2661                 self.streamP4FilesCb(entry)
2662
2663             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2664
2665             p4CmdList(["-x", "-", "print"],
2666                       stdin=fileArgs,
2667                       cb=streamP4FilesCbSelf)
2668
2669             # do the last chunk
2670             if self.stream_file.has_key('depotFile'):
2671                 self.streamOneP4File(self.stream_file, self.stream_contents)
2672
2673     def make_email(self, userid):
2674         if userid in self.users:
2675             return self.users[userid]
2676         else:
2677             return "%s <a@b>" % userid
2678
2679     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2680         """ Stream a p4 tag.
2681         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2682         """
2683
2684         if verbose:
2685             print "writing tag %s for commit %s" % (labelName, commit)
2686         gitStream.write("tag %s\n" % labelName)
2687         gitStream.write("from %s\n" % commit)
2688
2689         if labelDetails.has_key('Owner'):
2690             owner = labelDetails["Owner"]
2691         else:
2692             owner = None
2693
2694         # Try to use the owner of the p4 label, or failing that,
2695         # the current p4 user id.
2696         if owner:
2697             email = self.make_email(owner)
2698         else:
2699             email = self.make_email(self.p4UserId())
2700         tagger = "%s %s %s" % (email, epoch, self.tz)
2701
2702         gitStream.write("tagger %s\n" % tagger)
2703
2704         print "labelDetails=",labelDetails
2705         if labelDetails.has_key('Description'):
2706             description = labelDetails['Description']
2707         else:
2708             description = 'Label from git p4'
2709
2710         gitStream.write("data %d\n" % len(description))
2711         gitStream.write(description)
2712         gitStream.write("\n")
2713
2714     def inClientSpec(self, path):
2715         if not self.clientSpecDirs:
2716             return True
2717         inClientSpec = self.clientSpecDirs.map_in_client(path)
2718         if not inClientSpec and self.verbose:
2719             print('Ignoring file outside of client spec: {0}'.format(path))
2720         return inClientSpec
2721
2722     def hasBranchPrefix(self, path):
2723         if not self.branchPrefixes:
2724             return True
2725         hasPrefix = [p for p in self.branchPrefixes
2726                         if p4PathStartsWith(path, p)]
2727         if not hasPrefix and self.verbose:
2728             print('Ignoring file outside of prefix: {0}'.format(path))
2729         return hasPrefix
2730
2731     def commit(self, details, files, branch, parent = ""):
2732         epoch = details["time"]
2733         author = details["user"]
2734         jobs = self.extractJobsFromCommit(details)
2735
2736         if self.verbose:
2737             print('commit into {0}'.format(branch))
2738
2739         if self.clientSpecDirs:
2740             self.clientSpecDirs.update_client_spec_path_cache(files)
2741
2742         files = [f for f in files
2743             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2744
2745         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2746             print('Ignoring revision {0} as it would produce an empty commit.'
2747                 .format(details['change']))
2748             return
2749
2750         self.gitStream.write("commit %s\n" % branch)
2751         self.gitStream.write("mark :%s\n" % details["change"])
2752         self.committedChanges.add(int(details["change"]))
2753         committer = ""
2754         if author not in self.users:
2755             self.getUserMapFromPerforceServer()
2756         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2757
2758         self.gitStream.write("committer %s\n" % committer)
2759
2760         self.gitStream.write("data <<EOT\n")
2761         self.gitStream.write(details["desc"])
2762         if len(jobs) > 0:
2763             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2764         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2765                              (','.join(self.branchPrefixes), details["change"]))
2766         if len(details['options']) > 0:
2767             self.gitStream.write(": options = %s" % details['options'])
2768         self.gitStream.write("]\nEOT\n\n")
2769
2770         if len(parent) > 0:
2771             if self.verbose:
2772                 print "parent %s" % parent
2773             self.gitStream.write("from %s\n" % parent)
2774
2775         self.streamP4Files(files)
2776         self.gitStream.write("\n")
2777
2778         change = int(details["change"])
2779
2780         if self.labels.has_key(change):
2781             label = self.labels[change]
2782             labelDetails = label[0]
2783             labelRevisions = label[1]
2784             if self.verbose:
2785                 print "Change %s is labelled %s" % (change, labelDetails)
2786
2787             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2788                                                 for p in self.branchPrefixes])
2789
2790             if len(files) == len(labelRevisions):
2791
2792                 cleanedFiles = {}
2793                 for info in files:
2794                     if info["action"] in self.delete_actions:
2795                         continue
2796                     cleanedFiles[info["depotFile"]] = info["rev"]
2797
2798                 if cleanedFiles == labelRevisions:
2799                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2800
2801                 else:
2802                     if not self.silent:
2803                         print ("Tag %s does not match with change %s: files do not match."
2804                                % (labelDetails["label"], change))
2805
2806             else:
2807                 if not self.silent:
2808                     print ("Tag %s does not match with change %s: file count is different."
2809                            % (labelDetails["label"], change))
2810
2811     # Build a dictionary of changelists and labels, for "detect-labels" option.
2812     def getLabels(self):
2813         self.labels = {}
2814
2815         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2816         if len(l) > 0 and not self.silent:
2817             print "Finding files belonging to labels in %s" % `self.depotPaths`
2818
2819         for output in l:
2820             label = output["label"]
2821             revisions = {}
2822             newestChange = 0
2823             if self.verbose:
2824                 print "Querying files for label %s" % label
2825             for file in p4CmdList(["files"] +
2826                                       ["%s...@%s" % (p, label)
2827                                           for p in self.depotPaths]):
2828                 revisions[file["depotFile"]] = file["rev"]
2829                 change = int(file["change"])
2830                 if change > newestChange:
2831                     newestChange = change
2832
2833             self.labels[newestChange] = [output, revisions]
2834
2835         if self.verbose:
2836             print "Label changes: %s" % self.labels.keys()
2837
2838     # Import p4 labels as git tags. A direct mapping does not
2839     # exist, so assume that if all the files are at the same revision
2840     # then we can use that, or it's something more complicated we should
2841     # just ignore.
2842     def importP4Labels(self, stream, p4Labels):
2843         if verbose:
2844             print "import p4 labels: " + ' '.join(p4Labels)
2845
2846         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2847         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2848         if len(validLabelRegexp) == 0:
2849             validLabelRegexp = defaultLabelRegexp
2850         m = re.compile(validLabelRegexp)
2851
2852         for name in p4Labels:
2853             commitFound = False
2854
2855             if not m.match(name):
2856                 if verbose:
2857                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2858                 continue
2859
2860             if name in ignoredP4Labels:
2861                 continue
2862
2863             labelDetails = p4CmdList(['label', "-o", name])[0]
2864
2865             # get the most recent changelist for each file in this label
2866             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2867                                 for p in self.depotPaths])
2868
2869             if change.has_key('change'):
2870                 # find the corresponding git commit; take the oldest commit
2871                 changelist = int(change['change'])
2872                 if changelist in self.committedChanges:
2873                     gitCommit = ":%d" % changelist       # use a fast-import mark
2874                     commitFound = True
2875                 else:
2876                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2877                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2878                     if len(gitCommit) == 0:
2879                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2880                     else:
2881                         commitFound = True
2882                         gitCommit = gitCommit.strip()
2883
2884                 if commitFound:
2885                     # Convert from p4 time format
2886                     try:
2887                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2888                     except ValueError:
2889                         print "Could not convert label time %s" % labelDetails['Update']
2890                         tmwhen = 1
2891
2892                     when = int(time.mktime(tmwhen))
2893                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2894                     if verbose:
2895                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2896             else:
2897                 if verbose:
2898                     print "Label %s has no changelists - possibly deleted?" % name
2899
2900             if not commitFound:
2901                 # We can't import this label; don't try again as it will get very
2902                 # expensive repeatedly fetching all the files for labels that will
2903                 # never be imported. If the label is moved in the future, the
2904                 # ignore will need to be removed manually.
2905                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2906
2907     def guessProjectName(self):
2908         for p in self.depotPaths:
2909             if p.endswith("/"):
2910                 p = p[:-1]
2911             p = p[p.strip().rfind("/") + 1:]
2912             if not p.endswith("/"):
2913                p += "/"
2914             return p
2915
2916     def getBranchMapping(self):
2917         lostAndFoundBranches = set()
2918
2919         user = gitConfig("git-p4.branchUser")
2920         if len(user) > 0:
2921             command = "branches -u %s" % user
2922         else:
2923             command = "branches"
2924
2925         for info in p4CmdList(command):
2926             details = p4Cmd(["branch", "-o", info["branch"]])
2927             viewIdx = 0
2928             while details.has_key("View%s" % viewIdx):
2929                 paths = details["View%s" % viewIdx].split(" ")
2930                 viewIdx = viewIdx + 1
2931                 # require standard //depot/foo/... //depot/bar/... mapping
2932                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2933                     continue
2934                 source = paths[0]
2935                 destination = paths[1]
2936                 ## HACK
2937                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2938                     source = source[len(self.depotPaths[0]):-4]
2939                     destination = destination[len(self.depotPaths[0]):-4]
2940
2941                     if destination in self.knownBranches:
2942                         if not self.silent:
2943                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2944                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2945                         continue
2946
2947                     self.knownBranches[destination] = source
2948
2949                     lostAndFoundBranches.discard(destination)
2950
2951                     if source not in self.knownBranches:
2952                         lostAndFoundBranches.add(source)
2953
2954         # Perforce does not strictly require branches to be defined, so we also
2955         # check git config for a branch list.
2956         #
2957         # Example of branch definition in git config file:
2958         # [git-p4]
2959         #   branchList=main:branchA
2960         #   branchList=main:branchB
2961         #   branchList=branchA:branchC
2962         configBranches = gitConfigList("git-p4.branchList")
2963         for branch in configBranches:
2964             if branch:
2965                 (source, destination) = branch.split(":")
2966                 self.knownBranches[destination] = source
2967
2968                 lostAndFoundBranches.discard(destination)
2969
2970                 if source not in self.knownBranches:
2971                     lostAndFoundBranches.add(source)
2972
2973
2974         for branch in lostAndFoundBranches:
2975             self.knownBranches[branch] = branch
2976
2977     def getBranchMappingFromGitBranches(self):
2978         branches = p4BranchesInGit(self.importIntoRemotes)
2979         for branch in branches.keys():
2980             if branch == "master":
2981                 branch = "main"
2982             else:
2983                 branch = branch[len(self.projectName):]
2984             self.knownBranches[branch] = branch
2985
2986     def updateOptionDict(self, d):
2987         option_keys = {}
2988         if self.keepRepoPath:
2989             option_keys['keepRepoPath'] = 1
2990
2991         d["options"] = ' '.join(sorted(option_keys.keys()))
2992
2993     def readOptions(self, d):
2994         self.keepRepoPath = (d.has_key('options')
2995                              and ('keepRepoPath' in d['options']))
2996
2997     def gitRefForBranch(self, branch):
2998         if branch == "main":
2999             return self.refPrefix + "master"
3000
3001         if len(branch) <= 0:
3002             return branch
3003
3004         return self.refPrefix + self.projectName + branch
3005
3006     def gitCommitByP4Change(self, ref, change):
3007         if self.verbose:
3008             print "looking in ref " + ref + " for change %s using bisect..." % change
3009
3010         earliestCommit = ""
3011         latestCommit = parseRevision(ref)
3012
3013         while True:
3014             if self.verbose:
3015                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3016             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3017             if len(next) == 0:
3018                 if self.verbose:
3019                     print "argh"
3020                 return ""
3021             log = extractLogMessageFromGitCommit(next)
3022             settings = extractSettingsGitLog(log)
3023             currentChange = int(settings['change'])
3024             if self.verbose:
3025                 print "current change %s" % currentChange
3026
3027             if currentChange == change:
3028                 if self.verbose:
3029                     print "found %s" % next
3030                 return next
3031
3032             if currentChange < change:
3033                 earliestCommit = "^%s" % next
3034             else:
3035                 latestCommit = "%s" % next
3036
3037         return ""
3038
3039     def importNewBranch(self, branch, maxChange):
3040         # make fast-import flush all changes to disk and update the refs using the checkpoint
3041         # command so that we can try to find the branch parent in the git history
3042         self.gitStream.write("checkpoint\n\n");
3043         self.gitStream.flush();
3044         branchPrefix = self.depotPaths[0] + branch + "/"
3045         range = "@1,%s" % maxChange
3046         #print "prefix" + branchPrefix
3047         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3048         if len(changes) <= 0:
3049             return False
3050         firstChange = changes[0]
3051         #print "first change in branch: %s" % firstChange
3052         sourceBranch = self.knownBranches[branch]
3053         sourceDepotPath = self.depotPaths[0] + sourceBranch
3054         sourceRef = self.gitRefForBranch(sourceBranch)
3055         #print "source " + sourceBranch
3056
3057         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3058         #print "branch parent: %s" % branchParentChange
3059         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3060         if len(gitParent) > 0:
3061             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3062             #print "parent git commit: %s" % gitParent
3063
3064         self.importChanges(changes)
3065         return True
3066
3067     def searchParent(self, parent, branch, target):
3068         parentFound = False
3069         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3070                                      "--no-merges", parent]):
3071             blob = blob.strip()
3072             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3073                 parentFound = True
3074                 if self.verbose:
3075                     print "Found parent of %s in commit %s" % (branch, blob)
3076                 break
3077         if parentFound:
3078             return blob
3079         else:
3080             return None
3081
3082     def importChanges(self, changes):
3083         cnt = 1
3084         for change in changes:
3085             description = p4_describe(change)
3086             self.updateOptionDict(description)
3087
3088             if not self.silent:
3089                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3090                 sys.stdout.flush()
3091             cnt = cnt + 1
3092
3093             try:
3094                 if self.detectBranches:
3095                     branches = self.splitFilesIntoBranches(description)
3096                     for branch in branches.keys():
3097                         ## HACK  --hwn
3098                         branchPrefix = self.depotPaths[0] + branch + "/"
3099                         self.branchPrefixes = [ branchPrefix ]
3100
3101                         parent = ""
3102
3103                         filesForCommit = branches[branch]
3104
3105                         if self.verbose:
3106                             print "branch is %s" % branch
3107
3108                         self.updatedBranches.add(branch)
3109
3110                         if branch not in self.createdBranches:
3111                             self.createdBranches.add(branch)
3112                             parent = self.knownBranches[branch]
3113                             if parent == branch:
3114                                 parent = ""
3115                             else:
3116                                 fullBranch = self.projectName + branch
3117                                 if fullBranch not in self.p4BranchesInGit:
3118                                     if not self.silent:
3119                                         print("\n    Importing new branch %s" % fullBranch);
3120                                     if self.importNewBranch(branch, change - 1):
3121                                         parent = ""
3122                                         self.p4BranchesInGit.append(fullBranch)
3123                                     if not self.silent:
3124                                         print("\n    Resuming with change %s" % change);
3125
3126                                 if self.verbose:
3127                                     print "parent determined through known branches: %s" % parent
3128
3129                         branch = self.gitRefForBranch(branch)
3130                         parent = self.gitRefForBranch(parent)
3131
3132                         if self.verbose:
3133                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3134
3135                         if len(parent) == 0 and branch in self.initialParents:
3136                             parent = self.initialParents[branch]
3137                             del self.initialParents[branch]
3138
3139                         blob = None
3140                         if len(parent) > 0:
3141                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3142                             if self.verbose:
3143                                 print "Creating temporary branch: " + tempBranch
3144                             self.commit(description, filesForCommit, tempBranch)
3145                             self.tempBranches.append(tempBranch)
3146                             self.checkpoint()
3147                             blob = self.searchParent(parent, branch, tempBranch)
3148                         if blob:
3149                             self.commit(description, filesForCommit, branch, blob)
3150                         else:
3151                             if self.verbose:
3152                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3153                             self.commit(description, filesForCommit, branch, parent)
3154                 else:
3155                     files = self.extractFilesFromCommit(description)
3156                     self.commit(description, files, self.branch,
3157                                 self.initialParent)
3158                     # only needed once, to connect to the previous commit
3159                     self.initialParent = ""
3160             except IOError:
3161                 print self.gitError.read()
3162                 sys.exit(1)
3163
3164     def importHeadRevision(self, revision):
3165         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3166
3167         details = {}
3168         details["user"] = "git perforce import user"
3169         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3170                            % (' '.join(self.depotPaths), revision))
3171         details["change"] = revision
3172         newestRevision = 0
3173
3174         fileCnt = 0
3175         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3176
3177         for info in p4CmdList(["files"] + fileArgs):
3178
3179             if 'code' in info and info['code'] == 'error':
3180                 sys.stderr.write("p4 returned an error: %s\n"
3181                                  % info['data'])
3182                 if info['data'].find("must refer to client") >= 0:
3183                     sys.stderr.write("This particular p4 error is misleading.\n")
3184                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3185                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3186                 sys.exit(1)
3187             if 'p4ExitCode' in info:
3188                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3189                 sys.exit(1)
3190
3191
3192             change = int(info["change"])
3193             if change > newestRevision:
3194                 newestRevision = change
3195
3196             if info["action"] in self.delete_actions:
3197                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3198                 #fileCnt = fileCnt + 1
3199                 continue
3200
3201             for prop in ["depotFile", "rev", "action", "type" ]:
3202                 details["%s%s" % (prop, fileCnt)] = info[prop]
3203
3204             fileCnt = fileCnt + 1
3205
3206         details["change"] = newestRevision
3207
3208         # Use time from top-most change so that all git p4 clones of
3209         # the same p4 repo have the same commit SHA1s.
3210         res = p4_describe(newestRevision)
3211         details["time"] = res["time"]
3212
3213         self.updateOptionDict(details)
3214         try:
3215             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3216         except IOError:
3217             print "IO error with git fast-import. Is your git version recent enough?"
3218             print self.gitError.read()
3219
3220
3221     def run(self, args):
3222         self.depotPaths = []
3223         self.changeRange = ""
3224         self.previousDepotPaths = []
3225         self.hasOrigin = False
3226
3227         # map from branch depot path to parent branch
3228         self.knownBranches = {}
3229         self.initialParents = {}
3230
3231         if self.importIntoRemotes:
3232             self.refPrefix = "refs/remotes/p4/"
3233         else:
3234             self.refPrefix = "refs/heads/p4/"
3235
3236         if self.syncWithOrigin:
3237             self.hasOrigin = originP4BranchesExist()
3238             if self.hasOrigin:
3239                 if not self.silent:
3240                     print 'Syncing with origin first, using "git fetch origin"'
3241                 system("git fetch origin")
3242
3243         branch_arg_given = bool(self.branch)
3244         if len(self.branch) == 0:
3245             self.branch = self.refPrefix + "master"
3246             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3247                 system("git update-ref %s refs/heads/p4" % self.branch)
3248                 system("git branch -D p4")
3249
3250         # accept either the command-line option, or the configuration variable
3251         if self.useClientSpec:
3252             # will use this after clone to set the variable
3253             self.useClientSpec_from_options = True
3254         else:
3255             if gitConfigBool("git-p4.useclientspec"):
3256                 self.useClientSpec = True
3257         if self.useClientSpec:
3258             self.clientSpecDirs = getClientSpec()
3259
3260         # TODO: should always look at previous commits,
3261         # merge with previous imports, if possible.
3262         if args == []:
3263             if self.hasOrigin:
3264                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3265
3266             # branches holds mapping from branch name to sha1
3267             branches = p4BranchesInGit(self.importIntoRemotes)
3268
3269             # restrict to just this one, disabling detect-branches
3270             if branch_arg_given:
3271                 short = self.branch.split("/")[-1]
3272                 if short in branches:
3273                     self.p4BranchesInGit = [ short ]
3274             else:
3275                 self.p4BranchesInGit = branches.keys()
3276
3277             if len(self.p4BranchesInGit) > 1:
3278                 if not self.silent:
3279                     print "Importing from/into multiple branches"
3280                 self.detectBranches = True
3281                 for branch in branches.keys():
3282                     self.initialParents[self.refPrefix + branch] = \
3283                         branches[branch]
3284
3285             if self.verbose:
3286                 print "branches: %s" % self.p4BranchesInGit
3287
3288             p4Change = 0
3289             for branch in self.p4BranchesInGit:
3290                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3291
3292                 settings = extractSettingsGitLog(logMsg)
3293
3294                 self.readOptions(settings)
3295                 if (settings.has_key('depot-paths')
3296                     and settings.has_key ('change')):
3297                     change = int(settings['change']) + 1
3298                     p4Change = max(p4Change, change)
3299
3300                     depotPaths = sorted(settings['depot-paths'])
3301                     if self.previousDepotPaths == []:
3302                         self.previousDepotPaths = depotPaths
3303                     else:
3304                         paths = []
3305                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3306                             prev_list = prev.split("/")
3307                             cur_list = cur.split("/")
3308                             for i in range(0, min(len(cur_list), len(prev_list))):
3309                                 if cur_list[i] <> prev_list[i]:
3310                                     i = i - 1
3311                                     break
3312
3313                             paths.append ("/".join(cur_list[:i + 1]))
3314
3315                         self.previousDepotPaths = paths
3316
3317             if p4Change > 0:
3318                 self.depotPaths = sorted(self.previousDepotPaths)
3319                 self.changeRange = "@%s,#head" % p4Change
3320                 if not self.silent and not self.detectBranches:
3321                     print "Performing incremental import into %s git branch" % self.branch
3322
3323         # accept multiple ref name abbreviations:
3324         #    refs/foo/bar/branch -> use it exactly
3325         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3326         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3327         if not self.branch.startswith("refs/"):
3328             if self.importIntoRemotes:
3329                 prepend = "refs/remotes/"
3330             else:
3331                 prepend = "refs/heads/"
3332             if not self.branch.startswith("p4/"):
3333                 prepend += "p4/"
3334             self.branch = prepend + self.branch
3335
3336         if len(args) == 0 and self.depotPaths:
3337             if not self.silent:
3338                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3339         else:
3340             if self.depotPaths and self.depotPaths != args:
3341                 print ("previous import used depot path %s and now %s was specified. "
3342                        "This doesn't work!" % (' '.join (self.depotPaths),
3343                                                ' '.join (args)))
3344                 sys.exit(1)
3345
3346             self.depotPaths = sorted(args)
3347
3348         revision = ""
3349         self.users = {}
3350
3351         # Make sure no revision specifiers are used when --changesfile
3352         # is specified.
3353         bad_changesfile = False
3354         if len(self.changesFile) > 0:
3355             for p in self.depotPaths:
3356                 if p.find("@") >= 0 or p.find("#") >= 0:
3357                     bad_changesfile = True
3358                     break
3359         if bad_changesfile:
3360             die("Option --changesfile is incompatible with revision specifiers")
3361
3362         newPaths = []
3363         for p in self.depotPaths:
3364             if p.find("@") != -1:
3365                 atIdx = p.index("@")
3366                 self.changeRange = p[atIdx:]
3367                 if self.changeRange == "@all":
3368                     self.changeRange = ""
3369                 elif ',' not in self.changeRange:
3370                     revision = self.changeRange
3371                     self.changeRange = ""
3372                 p = p[:atIdx]
3373             elif p.find("#") != -1:
3374                 hashIdx = p.index("#")
3375                 revision = p[hashIdx:]
3376                 p = p[:hashIdx]
3377             elif self.previousDepotPaths == []:
3378                 # pay attention to changesfile, if given, else import
3379                 # the entire p4 tree at the head revision
3380                 if len(self.changesFile) == 0:
3381                     revision = "#head"
3382
3383             p = re.sub ("\.\.\.$", "", p)
3384             if not p.endswith("/"):
3385                 p += "/"
3386
3387             newPaths.append(p)
3388
3389         self.depotPaths = newPaths
3390
3391         # --detect-branches may change this for each branch
3392         self.branchPrefixes = self.depotPaths
3393
3394         self.loadUserMapFromCache()
3395         self.labels = {}
3396         if self.detectLabels:
3397             self.getLabels();
3398
3399         if self.detectBranches:
3400             ## FIXME - what's a P4 projectName ?
3401             self.projectName = self.guessProjectName()
3402
3403             if self.hasOrigin:
3404                 self.getBranchMappingFromGitBranches()
3405             else:
3406                 self.getBranchMapping()
3407             if self.verbose:
3408                 print "p4-git branches: %s" % self.p4BranchesInGit
3409                 print "initial parents: %s" % self.initialParents
3410             for b in self.p4BranchesInGit:
3411                 if b != "master":
3412
3413                     ## FIXME
3414                     b = b[len(self.projectName):]
3415                 self.createdBranches.add(b)
3416
3417         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3418
3419         self.importProcess = subprocess.Popen(["git", "fast-import"],
3420                                               stdin=subprocess.PIPE,
3421                                               stdout=subprocess.PIPE,
3422                                               stderr=subprocess.PIPE);
3423         self.gitOutput = self.importProcess.stdout
3424         self.gitStream = self.importProcess.stdin
3425         self.gitError = self.importProcess.stderr
3426
3427         if revision:
3428             self.importHeadRevision(revision)
3429         else:
3430             changes = []
3431
3432             if len(self.changesFile) > 0:
3433                 output = open(self.changesFile).readlines()
3434                 changeSet = set()
3435                 for line in output:
3436                     changeSet.add(int(line))
3437
3438                 for change in changeSet:
3439                     changes.append(change)
3440
3441                 changes.sort()
3442             else:
3443                 # catch "git p4 sync" with no new branches, in a repo that
3444                 # does not have any existing p4 branches
3445                 if len(args) == 0:
3446                     if not self.p4BranchesInGit:
3447                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3448
3449                     # The default branch is master, unless --branch is used to
3450                     # specify something else.  Make sure it exists, or complain
3451                     # nicely about how to use --branch.
3452                     if not self.detectBranches:
3453                         if not branch_exists(self.branch):
3454                             if branch_arg_given:
3455                                 die("Error: branch %s does not exist." % self.branch)
3456                             else:
3457                                 die("Error: no branch %s; perhaps specify one with --branch." %
3458                                     self.branch)
3459
3460                 if self.verbose:
3461                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3462                                                               self.changeRange)
3463                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3464
3465                 if len(self.maxChanges) > 0:
3466                     changes = changes[:min(int(self.maxChanges), len(changes))]
3467
3468             if len(changes) == 0:
3469                 if not self.silent:
3470                     print "No changes to import!"
3471             else:
3472                 if not self.silent and not self.detectBranches:
3473                     print "Import destination: %s" % self.branch
3474
3475                 self.updatedBranches = set()
3476
3477                 if not self.detectBranches:
3478                     if args:
3479                         # start a new branch
3480                         self.initialParent = ""
3481                     else:
3482                         # build on a previous revision
3483                         self.initialParent = parseRevision(self.branch)
3484
3485                 self.importChanges(changes)
3486
3487                 if not self.silent:
3488                     print ""
3489                     if len(self.updatedBranches) > 0:
3490                         sys.stdout.write("Updated branches: ")
3491                         for b in self.updatedBranches:
3492                             sys.stdout.write("%s " % b)
3493                         sys.stdout.write("\n")
3494
3495         if gitConfigBool("git-p4.importLabels"):
3496             self.importLabels = True
3497
3498         if self.importLabels:
3499             p4Labels = getP4Labels(self.depotPaths)
3500             gitTags = getGitTags()
3501
3502             missingP4Labels = p4Labels - gitTags
3503             self.importP4Labels(self.gitStream, missingP4Labels)
3504
3505         self.gitStream.close()
3506         if self.importProcess.wait() != 0:
3507             die("fast-import failed: %s" % self.gitError.read())
3508         self.gitOutput.close()
3509         self.gitError.close()
3510
3511         # Cleanup temporary branches created during import
3512         if self.tempBranches != []:
3513             for branch in self.tempBranches:
3514                 read_pipe("git update-ref -d %s" % branch)
3515             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3516
3517         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3518         # a convenient shortcut refname "p4".
3519         if self.importIntoRemotes:
3520             head_ref = self.refPrefix + "HEAD"
3521             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3522                 system(["git", "symbolic-ref", head_ref, self.branch])
3523
3524         return True
3525
3526 class P4Rebase(Command):
3527     def __init__(self):
3528         Command.__init__(self)
3529         self.options = [
3530                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3531         ]
3532         self.importLabels = False
3533         self.description = ("Fetches the latest revision from perforce and "
3534                             + "rebases the current work (branch) against it")
3535
3536     def run(self, args):
3537         sync = P4Sync()
3538         sync.importLabels = self.importLabels
3539         sync.run([])
3540
3541         return self.rebase()
3542
3543     def rebase(self):
3544         if os.system("git update-index --refresh") != 0:
3545             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3546         if len(read_pipe("git diff-index HEAD --")) > 0:
3547             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3548
3549         [upstream, settings] = findUpstreamBranchPoint()
3550         if len(upstream) == 0:
3551             die("Cannot find upstream branchpoint for rebase")
3552
3553         # the branchpoint may be p4/foo~3, so strip off the parent
3554         upstream = re.sub("~[0-9]+$", "", upstream)
3555
3556         print "Rebasing the current branch onto %s" % upstream
3557         oldHead = read_pipe("git rev-parse HEAD").strip()
3558         system("git rebase %s" % upstream)
3559         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3560         return True
3561
3562 class P4Clone(P4Sync):
3563     def __init__(self):
3564         P4Sync.__init__(self)
3565         self.description = "Creates a new git repository and imports from Perforce into it"
3566         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3567         self.options += [
3568             optparse.make_option("--destination", dest="cloneDestination",
3569                                  action='store', default=None,
3570                                  help="where to leave result of the clone"),
3571             optparse.make_option("--bare", dest="cloneBare",
3572                                  action="store_true", default=False),
3573         ]
3574         self.cloneDestination = None
3575         self.needsGit = False
3576         self.cloneBare = False
3577
3578     def defaultDestination(self, args):
3579         ## TODO: use common prefix of args?
3580         depotPath = args[0]
3581         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3582         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3583         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3584         depotDir = re.sub(r"/$", "", depotDir)
3585         return os.path.split(depotDir)[1]
3586
3587     def run(self, args):
3588         if len(args) < 1:
3589             return False
3590
3591         if self.keepRepoPath and not self.cloneDestination:
3592             sys.stderr.write("Must specify destination for --keep-path\n")
3593             sys.exit(1)
3594
3595         depotPaths = args
3596
3597         if not self.cloneDestination and len(depotPaths) > 1:
3598             self.cloneDestination = depotPaths[-1]
3599             depotPaths = depotPaths[:-1]
3600
3601         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3602         for p in depotPaths:
3603             if not p.startswith("//"):
3604                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3605                 return False
3606
3607         if not self.cloneDestination:
3608             self.cloneDestination = self.defaultDestination(args)
3609
3610         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3611
3612         if not os.path.exists(self.cloneDestination):
3613             os.makedirs(self.cloneDestination)
3614         chdir(self.cloneDestination)
3615
3616         init_cmd = [ "git", "init" ]
3617         if self.cloneBare:
3618             init_cmd.append("--bare")
3619         retcode = subprocess.call(init_cmd)
3620         if retcode:
3621             raise CalledProcessError(retcode, init_cmd)
3622
3623         if not P4Sync.run(self, depotPaths):
3624             return False
3625
3626         # create a master branch and check out a work tree
3627         if gitBranchExists(self.branch):
3628             system([ "git", "branch", "master", self.branch ])
3629             if not self.cloneBare:
3630                 system([ "git", "checkout", "-f" ])
3631         else:
3632             print 'Not checking out any branch, use ' \
3633                   '"git checkout -q -b master <branch>"'
3634
3635         # auto-set this variable if invoked with --use-client-spec
3636         if self.useClientSpec_from_options:
3637             system("git config --bool git-p4.useclientspec true")
3638
3639         return True
3640
3641 class P4Branches(Command):
3642     def __init__(self):
3643         Command.__init__(self)
3644         self.options = [ ]
3645         self.description = ("Shows the git branches that hold imports and their "
3646                             + "corresponding perforce depot paths")
3647         self.verbose = False
3648
3649     def run(self, args):
3650         if originP4BranchesExist():
3651             createOrUpdateBranchesFromOrigin()
3652
3653         cmdline = "git rev-parse --symbolic "
3654         cmdline += " --remotes"
3655
3656         for line in read_pipe_lines(cmdline):
3657             line = line.strip()
3658
3659             if not line.startswith('p4/') or line == "p4/HEAD":
3660                 continue
3661             branch = line
3662
3663             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3664             settings = extractSettingsGitLog(log)
3665
3666             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3667         return True
3668
3669 class HelpFormatter(optparse.IndentedHelpFormatter):
3670     def __init__(self):
3671         optparse.IndentedHelpFormatter.__init__(self)
3672
3673     def format_description(self, description):
3674         if description:
3675             return description + "\n"
3676         else:
3677             return ""
3678
3679 def printUsage(commands):
3680     print "usage: %s <command> [options]" % sys.argv[0]
3681     print ""
3682     print "valid commands: %s" % ", ".join(commands)
3683     print ""
3684     print "Try %s <command> --help for command specific help." % sys.argv[0]
3685     print ""
3686
3687 commands = {
3688     "debug" : P4Debug,
3689     "submit" : P4Submit,
3690     "commit" : P4Submit,
3691     "sync" : P4Sync,
3692     "rebase" : P4Rebase,
3693     "clone" : P4Clone,
3694     "rollback" : P4RollBack,
3695     "branches" : P4Branches
3696 }
3697
3698
3699 def main():
3700     if len(sys.argv[1:]) == 0:
3701         printUsage(commands.keys())
3702         sys.exit(2)
3703
3704     cmdName = sys.argv[1]
3705     try:
3706         klass = commands[cmdName]
3707         cmd = klass()
3708     except KeyError:
3709         print "unknown command %s" % cmdName
3710         print ""
3711         printUsage(commands.keys())
3712         sys.exit(2)
3713
3714     options = cmd.options
3715     cmd.gitdir = os.environ.get("GIT_DIR", None)
3716
3717     args = sys.argv[2:]
3718
3719     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3720     if cmd.needsGit:
3721         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3722
3723     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3724                                    options,
3725                                    description = cmd.description,
3726                                    formatter = HelpFormatter())
3727
3728     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3729     global verbose
3730     verbose = cmd.verbose
3731     if cmd.needsGit:
3732         if cmd.gitdir == None:
3733             cmd.gitdir = os.path.abspath(".git")
3734             if not isValidGitDir(cmd.gitdir):
3735                 # "rev-parse --git-dir" without arguments will try $PWD/.git
3736                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3737                 if os.path.exists(cmd.gitdir):
3738                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3739                     if len(cdup) > 0:
3740                         chdir(cdup);
3741
3742         if not isValidGitDir(cmd.gitdir):
3743             if isValidGitDir(cmd.gitdir + "/.git"):
3744                 cmd.gitdir += "/.git"
3745             else:
3746                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3747
3748         # so git commands invoked from the P4 workspace will succeed
3749         os.environ["GIT_DIR"] = cmd.gitdir
3750
3751     if not cmd.run(args):
3752         parser.print_help()
3753         sys.exit(2)
3754
3755
3756 if __name__ == '__main__':
3757     main()