ref-filter: add --no-contains option to tag/branch/for-each-ref
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31     from subprocess import CalledProcessError
32 except ImportError:
33     # from python2.7:subprocess.py
34     # Exception classes used by this module.
35     class CalledProcessError(Exception):
36         """This exception is raised when a process run by check_call() returns
37         a non-zero exit status.  The exit status will be stored in the
38         returncode attribute."""
39         def __init__(self, returncode, cmd):
40             self.returncode = returncode
41             self.cmd = cmd
42         def __str__(self):
43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54     """Build a suitable p4 command line.
55
56     This consolidates building and returning a p4 command line into one
57     location. It means that hooking into the environment, or other configuration
58     can be done more easily.
59     """
60     real_cmd = ["p4"]
61
62     user = gitConfig("git-p4.user")
63     if len(user) > 0:
64         real_cmd += ["-u",user]
65
66     password = gitConfig("git-p4.password")
67     if len(password) > 0:
68         real_cmd += ["-P", password]
69
70     port = gitConfig("git-p4.port")
71     if len(port) > 0:
72         real_cmd += ["-p", port]
73
74     host = gitConfig("git-p4.host")
75     if len(host) > 0:
76         real_cmd += ["-H", host]
77
78     client = gitConfig("git-p4.client")
79     if len(client) > 0:
80         real_cmd += ["-c", client]
81
82     retries = gitConfigInt("git-p4.retries")
83     if retries is None:
84         # Perform 3 retries by default
85         retries = 3
86     if retries > 0:
87         # Provide a way to not pass this option by setting git-p4.retries to 0
88         real_cmd += ["-r", str(retries)]
89
90     if isinstance(cmd,basestring):
91         real_cmd = ' '.join(real_cmd) + ' ' + cmd
92     else:
93         real_cmd += cmd
94     return real_cmd
95
96 def git_dir(path):
97     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98         This won't automatically add ".git" to a directory.
99     """
100     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101     if not d or len(d) == 0:
102         return None
103     else:
104         return d
105
106 def chdir(path, is_client_path=False):
107     """Do chdir to the given path, and set the PWD environment
108        variable for use by P4.  It does not look at getcwd() output.
109        Since we're not using the shell, it is necessary to set the
110        PWD environment variable explicitly.
111
112        Normally, expand the path to force it to be absolute.  This
113        addresses the use of relative path names inside P4 settings,
114        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
115        as given; it looks for .p4config using PWD.
116
117        If is_client_path, the path was handed to us directly by p4,
118        and may be a symbolic link.  Do not call os.getcwd() in this
119        case, because it will cause p4 to think that PWD is not inside
120        the client path.
121        """
122
123     os.chdir(path)
124     if not is_client_path:
125         path = os.getcwd()
126     os.environ['PWD'] = path
127
128 def calcDiskFree():
129     """Return free space in bytes on the disk of the given dirname."""
130     if platform.system() == 'Windows':
131         free_bytes = ctypes.c_ulonglong(0)
132         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133         return free_bytes.value
134     else:
135         st = os.statvfs(os.getcwd())
136         return st.f_bavail * st.f_frsize
137
138 def die(msg):
139     if verbose:
140         raise Exception(msg)
141     else:
142         sys.stderr.write(msg + "\n")
143         sys.exit(1)
144
145 def write_pipe(c, stdin):
146     if verbose:
147         sys.stderr.write('Writing pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151     pipe = p.stdin
152     val = pipe.write(stdin)
153     pipe.close()
154     if p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_write_pipe(c, stdin):
160     real_cmd = p4_build_cmd(c)
161     return write_pipe(real_cmd, stdin)
162
163 def read_pipe(c, ignore_error=False):
164     if verbose:
165         sys.stderr.write('Reading pipe: %s\n' % str(c))
166
167     expand = isinstance(c,basestring)
168     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
169     (out, err) = p.communicate()
170     if p.returncode != 0 and not ignore_error:
171         die('Command failed: %s\nError: %s' % (str(c), err))
172     return out
173
174 def p4_read_pipe(c, ignore_error=False):
175     real_cmd = p4_build_cmd(c)
176     return read_pipe(real_cmd, ignore_error)
177
178 def read_pipe_lines(c):
179     if verbose:
180         sys.stderr.write('Reading pipe: %s\n' % str(c))
181
182     expand = isinstance(c, basestring)
183     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
184     pipe = p.stdout
185     val = pipe.readlines()
186     if pipe.close() or p.wait():
187         die('Command failed: %s' % str(c))
188
189     return val
190
191 def p4_read_pipe_lines(c):
192     """Specifically invoke p4 on the command supplied. """
193     real_cmd = p4_build_cmd(c)
194     return read_pipe_lines(real_cmd)
195
196 def p4_has_command(cmd):
197     """Ask p4 for help on this command.  If it returns an error, the
198        command does not exist in this version of p4."""
199     real_cmd = p4_build_cmd(["help", cmd])
200     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
201                                    stderr=subprocess.PIPE)
202     p.communicate()
203     return p.returncode == 0
204
205 def p4_has_move_command():
206     """See if the move command exists, that it supports -k, and that
207        it has not been administratively disabled.  The arguments
208        must be correct, but the filenames do not have to exist.  Use
209        ones with wildcards so even if they exist, it will fail."""
210
211     if not p4_has_command("move"):
212         return False
213     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
214     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
215     (out, err) = p.communicate()
216     # return code will be 1 in either case
217     if err.find("Invalid option") >= 0:
218         return False
219     if err.find("disabled") >= 0:
220         return False
221     # assume it failed because @... was invalid changelist
222     return True
223
224 def system(cmd, ignore_error=False):
225     expand = isinstance(cmd,basestring)
226     if verbose:
227         sys.stderr.write("executing %s\n" % str(cmd))
228     retcode = subprocess.call(cmd, shell=expand)
229     if retcode and not ignore_error:
230         raise CalledProcessError(retcode, cmd)
231
232     return retcode
233
234 def p4_system(cmd):
235     """Specifically invoke p4 as the system command. """
236     real_cmd = p4_build_cmd(cmd)
237     expand = isinstance(real_cmd, basestring)
238     retcode = subprocess.call(real_cmd, shell=expand)
239     if retcode:
240         raise CalledProcessError(retcode, real_cmd)
241
242 _p4_version_string = None
243 def p4_version_string():
244     """Read the version string, showing just the last line, which
245        hopefully is the interesting version bit.
246
247        $ p4 -V
248        Perforce - The Fast Software Configuration Management System.
249        Copyright 1995-2011 Perforce Software.  All rights reserved.
250        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
251     """
252     global _p4_version_string
253     if not _p4_version_string:
254         a = p4_read_pipe_lines(["-V"])
255         _p4_version_string = a[-1].rstrip()
256     return _p4_version_string
257
258 def p4_integrate(src, dest):
259     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
260
261 def p4_sync(f, *options):
262     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
263
264 def p4_add(f):
265     # forcibly add file names with wildcards
266     if wildcard_present(f):
267         p4_system(["add", "-f", f])
268     else:
269         p4_system(["add", f])
270
271 def p4_delete(f):
272     p4_system(["delete", wildcard_encode(f)])
273
274 def p4_edit(f, *options):
275     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
276
277 def p4_revert(f):
278     p4_system(["revert", wildcard_encode(f)])
279
280 def p4_reopen(type, f):
281     p4_system(["reopen", "-t", type, wildcard_encode(f)])
282
283 def p4_reopen_in_change(changelist, files):
284     cmd = ["reopen", "-c", str(changelist)] + files
285     p4_system(cmd)
286
287 def p4_move(src, dest):
288     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
289
290 def p4_last_change():
291     results = p4CmdList(["changes", "-m", "1"])
292     return int(results[0]['change'])
293
294 def p4_describe(change):
295     """Make sure it returns a valid result by checking for
296        the presence of field "time".  Return a dict of the
297        results."""
298
299     ds = p4CmdList(["describe", "-s", str(change)])
300     if len(ds) != 1:
301         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
302
303     d = ds[0]
304
305     if "p4ExitCode" in d:
306         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
307                                                       str(d)))
308     if "code" in d:
309         if d["code"] == "error":
310             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
311
312     if "time" not in d:
313         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
314
315     return d
316
317 #
318 # Canonicalize the p4 type and return a tuple of the
319 # base type, plus any modifiers.  See "p4 help filetypes"
320 # for a list and explanation.
321 #
322 def split_p4_type(p4type):
323
324     p4_filetypes_historical = {
325         "ctempobj": "binary+Sw",
326         "ctext": "text+C",
327         "cxtext": "text+Cx",
328         "ktext": "text+k",
329         "kxtext": "text+kx",
330         "ltext": "text+F",
331         "tempobj": "binary+FSw",
332         "ubinary": "binary+F",
333         "uresource": "resource+F",
334         "uxbinary": "binary+Fx",
335         "xbinary": "binary+x",
336         "xltext": "text+Fx",
337         "xtempobj": "binary+Swx",
338         "xtext": "text+x",
339         "xunicode": "unicode+x",
340         "xutf16": "utf16+x",
341     }
342     if p4type in p4_filetypes_historical:
343         p4type = p4_filetypes_historical[p4type]
344     mods = ""
345     s = p4type.split("+")
346     base = s[0]
347     mods = ""
348     if len(s) > 1:
349         mods = s[1]
350     return (base, mods)
351
352 #
353 # return the raw p4 type of a file (text, text+ko, etc)
354 #
355 def p4_type(f):
356     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
357     return results[0]['headType']
358
359 #
360 # Given a type base and modifier, return a regexp matching
361 # the keywords that can be expanded in the file
362 #
363 def p4_keywords_regexp_for_type(base, type_mods):
364     if base in ("text", "unicode", "binary"):
365         kwords = None
366         if "ko" in type_mods:
367             kwords = 'Id|Header'
368         elif "k" in type_mods:
369             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
370         else:
371             return None
372         pattern = r"""
373             \$              # Starts with a dollar, followed by...
374             (%s)            # one of the keywords, followed by...
375             (:[^$\n]+)?     # possibly an old expansion, followed by...
376             \$              # another dollar
377             """ % kwords
378         return pattern
379     else:
380         return None
381
382 #
383 # Given a file, return a regexp matching the possible
384 # RCS keywords that will be expanded, or None for files
385 # with kw expansion turned off.
386 #
387 def p4_keywords_regexp_for_file(file):
388     if not os.path.exists(file):
389         return None
390     else:
391         (type_base, type_mods) = split_p4_type(p4_type(file))
392         return p4_keywords_regexp_for_type(type_base, type_mods)
393
394 def setP4ExecBit(file, mode):
395     # Reopens an already open file and changes the execute bit to match
396     # the execute bit setting in the passed in mode.
397
398     p4Type = "+x"
399
400     if not isModeExec(mode):
401         p4Type = getP4OpenedType(file)
402         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
403         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
404         if p4Type[-1] == "+":
405             p4Type = p4Type[0:-1]
406
407     p4_reopen(p4Type, file)
408
409 def getP4OpenedType(file):
410     # Returns the perforce file type for the given file.
411
412     result = p4_read_pipe(["opened", wildcard_encode(file)])
413     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
414     if match:
415         return match.group(1)
416     else:
417         die("Could not determine file type for %s (result: '%s')" % (file, result))
418
419 # Return the set of all p4 labels
420 def getP4Labels(depotPaths):
421     labels = set()
422     if isinstance(depotPaths,basestring):
423         depotPaths = [depotPaths]
424
425     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
426         label = l['label']
427         labels.add(label)
428
429     return labels
430
431 # Return the set of all git tags
432 def getGitTags():
433     gitTags = set()
434     for line in read_pipe_lines(["git", "tag"]):
435         tag = line.strip()
436         gitTags.add(tag)
437     return gitTags
438
439 def diffTreePattern():
440     # This is a simple generator for the diff tree regex pattern. This could be
441     # a class variable if this and parseDiffTreeEntry were a part of a class.
442     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
443     while True:
444         yield pattern
445
446 def parseDiffTreeEntry(entry):
447     """Parses a single diff tree entry into its component elements.
448
449     See git-diff-tree(1) manpage for details about the format of the diff
450     output. This method returns a dictionary with the following elements:
451
452     src_mode - The mode of the source file
453     dst_mode - The mode of the destination file
454     src_sha1 - The sha1 for the source file
455     dst_sha1 - The sha1 fr the destination file
456     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
457     status_score - The score for the status (applicable for 'C' and 'R'
458                    statuses). This is None if there is no score.
459     src - The path for the source file.
460     dst - The path for the destination file. This is only present for
461           copy or renames. If it is not present, this is None.
462
463     If the pattern is not matched, None is returned."""
464
465     match = diffTreePattern().next().match(entry)
466     if match:
467         return {
468             'src_mode': match.group(1),
469             'dst_mode': match.group(2),
470             'src_sha1': match.group(3),
471             'dst_sha1': match.group(4),
472             'status': match.group(5),
473             'status_score': match.group(6),
474             'src': match.group(7),
475             'dst': match.group(10)
476         }
477     return None
478
479 def isModeExec(mode):
480     # Returns True if the given git mode represents an executable file,
481     # otherwise False.
482     return mode[-3:] == "755"
483
484 def isModeExecChanged(src_mode, dst_mode):
485     return isModeExec(src_mode) != isModeExec(dst_mode)
486
487 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
488
489     if isinstance(cmd,basestring):
490         cmd = "-G " + cmd
491         expand = True
492     else:
493         cmd = ["-G"] + cmd
494         expand = False
495
496     cmd = p4_build_cmd(cmd)
497     if verbose:
498         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
499
500     # Use a temporary file to avoid deadlocks without
501     # subprocess.communicate(), which would put another copy
502     # of stdout into memory.
503     stdin_file = None
504     if stdin is not None:
505         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
506         if isinstance(stdin,basestring):
507             stdin_file.write(stdin)
508         else:
509             for i in stdin:
510                 stdin_file.write(i + '\n')
511         stdin_file.flush()
512         stdin_file.seek(0)
513
514     p4 = subprocess.Popen(cmd,
515                           shell=expand,
516                           stdin=stdin_file,
517                           stdout=subprocess.PIPE)
518
519     result = []
520     try:
521         while True:
522             entry = marshal.load(p4.stdout)
523             if cb is not None:
524                 cb(entry)
525             else:
526                 result.append(entry)
527     except EOFError:
528         pass
529     exitCode = p4.wait()
530     if exitCode != 0:
531         entry = {}
532         entry["p4ExitCode"] = exitCode
533         result.append(entry)
534
535     return result
536
537 def p4Cmd(cmd):
538     list = p4CmdList(cmd)
539     result = {}
540     for entry in list:
541         result.update(entry)
542     return result;
543
544 def p4Where(depotPath):
545     if not depotPath.endswith("/"):
546         depotPath += "/"
547     depotPathLong = depotPath + "..."
548     outputList = p4CmdList(["where", depotPathLong])
549     output = None
550     for entry in outputList:
551         if "depotFile" in entry:
552             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
553             # The base path always ends with "/...".
554             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
555                 output = entry
556                 break
557         elif "data" in entry:
558             data = entry.get("data")
559             space = data.find(" ")
560             if data[:space] == depotPath:
561                 output = entry
562                 break
563     if output == None:
564         return ""
565     if output["code"] == "error":
566         return ""
567     clientPath = ""
568     if "path" in output:
569         clientPath = output.get("path")
570     elif "data" in output:
571         data = output.get("data")
572         lastSpace = data.rfind(" ")
573         clientPath = data[lastSpace + 1:]
574
575     if clientPath.endswith("..."):
576         clientPath = clientPath[:-3]
577     return clientPath
578
579 def currentGitBranch():
580     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
581     if retcode != 0:
582         # on a detached head
583         return None
584     else:
585         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
586
587 def isValidGitDir(path):
588     return git_dir(path) != None
589
590 def parseRevision(ref):
591     return read_pipe("git rev-parse %s" % ref).strip()
592
593 def branchExists(ref):
594     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
595                      ignore_error=True)
596     return len(rev) > 0
597
598 def extractLogMessageFromGitCommit(commit):
599     logMessage = ""
600
601     ## fixme: title is first line of commit, not 1st paragraph.
602     foundTitle = False
603     for log in read_pipe_lines("git cat-file commit %s" % commit):
604        if not foundTitle:
605            if len(log) == 1:
606                foundTitle = True
607            continue
608
609        logMessage += log
610     return logMessage
611
612 def extractSettingsGitLog(log):
613     values = {}
614     for line in log.split("\n"):
615         line = line.strip()
616         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
617         if not m:
618             continue
619
620         assignments = m.group(1).split (':')
621         for a in assignments:
622             vals = a.split ('=')
623             key = vals[0].strip()
624             val = ('='.join (vals[1:])).strip()
625             if val.endswith ('\"') and val.startswith('"'):
626                 val = val[1:-1]
627
628             values[key] = val
629
630     paths = values.get("depot-paths")
631     if not paths:
632         paths = values.get("depot-path")
633     if paths:
634         values['depot-paths'] = paths.split(',')
635     return values
636
637 def gitBranchExists(branch):
638     proc = subprocess.Popen(["git", "rev-parse", branch],
639                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
640     return proc.wait() == 0;
641
642 _gitConfig = {}
643
644 def gitConfig(key, typeSpecifier=None):
645     if not _gitConfig.has_key(key):
646         cmd = [ "git", "config" ]
647         if typeSpecifier:
648             cmd += [ typeSpecifier ]
649         cmd += [ key ]
650         s = read_pipe(cmd, ignore_error=True)
651         _gitConfig[key] = s.strip()
652     return _gitConfig[key]
653
654 def gitConfigBool(key):
655     """Return a bool, using git config --bool.  It is True only if the
656        variable is set to true, and False if set to false or not present
657        in the config."""
658
659     if not _gitConfig.has_key(key):
660         _gitConfig[key] = gitConfig(key, '--bool') == "true"
661     return _gitConfig[key]
662
663 def gitConfigInt(key):
664     if not _gitConfig.has_key(key):
665         cmd = [ "git", "config", "--int", key ]
666         s = read_pipe(cmd, ignore_error=True)
667         v = s.strip()
668         try:
669             _gitConfig[key] = int(gitConfig(key, '--int'))
670         except ValueError:
671             _gitConfig[key] = None
672     return _gitConfig[key]
673
674 def gitConfigList(key):
675     if not _gitConfig.has_key(key):
676         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
677         _gitConfig[key] = s.strip().splitlines()
678         if _gitConfig[key] == ['']:
679             _gitConfig[key] = []
680     return _gitConfig[key]
681
682 def p4BranchesInGit(branchesAreInRemotes=True):
683     """Find all the branches whose names start with "p4/", looking
684        in remotes or heads as specified by the argument.  Return
685        a dictionary of { branch: revision } for each one found.
686        The branch names are the short names, without any
687        "p4/" prefix."""
688
689     branches = {}
690
691     cmdline = "git rev-parse --symbolic "
692     if branchesAreInRemotes:
693         cmdline += "--remotes"
694     else:
695         cmdline += "--branches"
696
697     for line in read_pipe_lines(cmdline):
698         line = line.strip()
699
700         # only import to p4/
701         if not line.startswith('p4/'):
702             continue
703         # special symbolic ref to p4/master
704         if line == "p4/HEAD":
705             continue
706
707         # strip off p4/ prefix
708         branch = line[len("p4/"):]
709
710         branches[branch] = parseRevision(line)
711
712     return branches
713
714 def branch_exists(branch):
715     """Make sure that the given ref name really exists."""
716
717     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
718     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
719     out, _ = p.communicate()
720     if p.returncode:
721         return False
722     # expect exactly one line of output: the branch name
723     return out.rstrip() == branch
724
725 def findUpstreamBranchPoint(head = "HEAD"):
726     branches = p4BranchesInGit()
727     # map from depot-path to branch name
728     branchByDepotPath = {}
729     for branch in branches.keys():
730         tip = branches[branch]
731         log = extractLogMessageFromGitCommit(tip)
732         settings = extractSettingsGitLog(log)
733         if settings.has_key("depot-paths"):
734             paths = ",".join(settings["depot-paths"])
735             branchByDepotPath[paths] = "remotes/p4/" + branch
736
737     settings = None
738     parent = 0
739     while parent < 65535:
740         commit = head + "~%s" % parent
741         log = extractLogMessageFromGitCommit(commit)
742         settings = extractSettingsGitLog(log)
743         if settings.has_key("depot-paths"):
744             paths = ",".join(settings["depot-paths"])
745             if branchByDepotPath.has_key(paths):
746                 return [branchByDepotPath[paths], settings]
747
748         parent = parent + 1
749
750     return ["", settings]
751
752 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
753     if not silent:
754         print ("Creating/updating branch(es) in %s based on origin branch(es)"
755                % localRefPrefix)
756
757     originPrefix = "origin/p4/"
758
759     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
760         line = line.strip()
761         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
762             continue
763
764         headName = line[len(originPrefix):]
765         remoteHead = localRefPrefix + headName
766         originHead = line
767
768         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
769         if (not original.has_key('depot-paths')
770             or not original.has_key('change')):
771             continue
772
773         update = False
774         if not gitBranchExists(remoteHead):
775             if verbose:
776                 print "creating %s" % remoteHead
777             update = True
778         else:
779             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
780             if settings.has_key('change') > 0:
781                 if settings['depot-paths'] == original['depot-paths']:
782                     originP4Change = int(original['change'])
783                     p4Change = int(settings['change'])
784                     if originP4Change > p4Change:
785                         print ("%s (%s) is newer than %s (%s). "
786                                "Updating p4 branch from origin."
787                                % (originHead, originP4Change,
788                                   remoteHead, p4Change))
789                         update = True
790                 else:
791                     print ("Ignoring: %s was imported from %s while "
792                            "%s was imported from %s"
793                            % (originHead, ','.join(original['depot-paths']),
794                               remoteHead, ','.join(settings['depot-paths'])))
795
796         if update:
797             system("git update-ref %s %s" % (remoteHead, originHead))
798
799 def originP4BranchesExist():
800         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
801
802
803 def p4ParseNumericChangeRange(parts):
804     changeStart = int(parts[0][1:])
805     if parts[1] == '#head':
806         changeEnd = p4_last_change()
807     else:
808         changeEnd = int(parts[1])
809
810     return (changeStart, changeEnd)
811
812 def chooseBlockSize(blockSize):
813     if blockSize:
814         return blockSize
815     else:
816         return defaultBlockSize
817
818 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
819     assert depotPaths
820
821     # Parse the change range into start and end. Try to find integer
822     # revision ranges as these can be broken up into blocks to avoid
823     # hitting server-side limits (maxrows, maxscanresults). But if
824     # that doesn't work, fall back to using the raw revision specifier
825     # strings, without using block mode.
826
827     if changeRange is None or changeRange == '':
828         changeStart = 1
829         changeEnd = p4_last_change()
830         block_size = chooseBlockSize(requestedBlockSize)
831     else:
832         parts = changeRange.split(',')
833         assert len(parts) == 2
834         try:
835             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
836             block_size = chooseBlockSize(requestedBlockSize)
837         except:
838             changeStart = parts[0][1:]
839             changeEnd = parts[1]
840             if requestedBlockSize:
841                 die("cannot use --changes-block-size with non-numeric revisions")
842             block_size = None
843
844     changes = set()
845
846     # Retrieve changes a block at a time, to prevent running
847     # into a MaxResults/MaxScanRows error from the server.
848
849     while True:
850         cmd = ['changes']
851
852         if block_size:
853             end = min(changeEnd, changeStart + block_size)
854             revisionRange = "%d,%d" % (changeStart, end)
855         else:
856             revisionRange = "%s,%s" % (changeStart, changeEnd)
857
858         for p in depotPaths:
859             cmd += ["%s...@%s" % (p, revisionRange)]
860
861         # Insert changes in chronological order
862         for line in reversed(p4_read_pipe_lines(cmd)):
863             changes.add(int(line.split(" ")[1]))
864
865         if not block_size:
866             break
867
868         if end >= changeEnd:
869             break
870
871         changeStart = end + 1
872
873     changes = sorted(changes)
874     return changes
875
876 def p4PathStartsWith(path, prefix):
877     # This method tries to remedy a potential mixed-case issue:
878     #
879     # If UserA adds  //depot/DirA/file1
880     # and UserB adds //depot/dira/file2
881     #
882     # we may or may not have a problem. If you have core.ignorecase=true,
883     # we treat DirA and dira as the same directory
884     if gitConfigBool("core.ignorecase"):
885         return path.lower().startswith(prefix.lower())
886     return path.startswith(prefix)
887
888 def getClientSpec():
889     """Look at the p4 client spec, create a View() object that contains
890        all the mappings, and return it."""
891
892     specList = p4CmdList("client -o")
893     if len(specList) != 1:
894         die('Output from "client -o" is %d lines, expecting 1' %
895             len(specList))
896
897     # dictionary of all client parameters
898     entry = specList[0]
899
900     # the //client/ name
901     client_name = entry["Client"]
902
903     # just the keys that start with "View"
904     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
905
906     # hold this new View
907     view = View(client_name)
908
909     # append the lines, in order, to the view
910     for view_num in range(len(view_keys)):
911         k = "View%d" % view_num
912         if k not in view_keys:
913             die("Expected view key %s missing" % k)
914         view.append(entry[k])
915
916     return view
917
918 def getClientRoot():
919     """Grab the client directory."""
920
921     output = p4CmdList("client -o")
922     if len(output) != 1:
923         die('Output from "client -o" is %d lines, expecting 1' % len(output))
924
925     entry = output[0]
926     if "Root" not in entry:
927         die('Client has no "Root"')
928
929     return entry["Root"]
930
931 #
932 # P4 wildcards are not allowed in filenames.  P4 complains
933 # if you simply add them, but you can force it with "-f", in
934 # which case it translates them into %xx encoding internally.
935 #
936 def wildcard_decode(path):
937     # Search for and fix just these four characters.  Do % last so
938     # that fixing it does not inadvertently create new %-escapes.
939     # Cannot have * in a filename in windows; untested as to
940     # what p4 would do in such a case.
941     if not platform.system() == "Windows":
942         path = path.replace("%2A", "*")
943     path = path.replace("%23", "#") \
944                .replace("%40", "@") \
945                .replace("%25", "%")
946     return path
947
948 def wildcard_encode(path):
949     # do % first to avoid double-encoding the %s introduced here
950     path = path.replace("%", "%25") \
951                .replace("*", "%2A") \
952                .replace("#", "%23") \
953                .replace("@", "%40")
954     return path
955
956 def wildcard_present(path):
957     m = re.search("[*#@%]", path)
958     return m is not None
959
960 class LargeFileSystem(object):
961     """Base class for large file system support."""
962
963     def __init__(self, writeToGitStream):
964         self.largeFiles = set()
965         self.writeToGitStream = writeToGitStream
966
967     def generatePointer(self, cloneDestination, contentFile):
968         """Return the content of a pointer file that is stored in Git instead of
969            the actual content."""
970         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
971
972     def pushFile(self, localLargeFile):
973         """Push the actual content which is not stored in the Git repository to
974            a server."""
975         assert False, "Method 'pushFile' required in " + self.__class__.__name__
976
977     def hasLargeFileExtension(self, relPath):
978         return reduce(
979             lambda a, b: a or b,
980             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
981             False
982         )
983
984     def generateTempFile(self, contents):
985         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
986         for d in contents:
987             contentFile.write(d)
988         contentFile.close()
989         return contentFile.name
990
991     def exceedsLargeFileThreshold(self, relPath, contents):
992         if gitConfigInt('git-p4.largeFileThreshold'):
993             contentsSize = sum(len(d) for d in contents)
994             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
995                 return True
996         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
997             contentsSize = sum(len(d) for d in contents)
998             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
999                 return False
1000             contentTempFile = self.generateTempFile(contents)
1001             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1002             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1003             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1004             zf.close()
1005             compressedContentsSize = zf.infolist()[0].compress_size
1006             os.remove(contentTempFile)
1007             os.remove(compressedContentFile.name)
1008             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1009                 return True
1010         return False
1011
1012     def addLargeFile(self, relPath):
1013         self.largeFiles.add(relPath)
1014
1015     def removeLargeFile(self, relPath):
1016         self.largeFiles.remove(relPath)
1017
1018     def isLargeFile(self, relPath):
1019         return relPath in self.largeFiles
1020
1021     def processContent(self, git_mode, relPath, contents):
1022         """Processes the content of git fast import. This method decides if a
1023            file is stored in the large file system and handles all necessary
1024            steps."""
1025         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1026             contentTempFile = self.generateTempFile(contents)
1027             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1028             if pointer_git_mode:
1029                 git_mode = pointer_git_mode
1030             if localLargeFile:
1031                 # Move temp file to final location in large file system
1032                 largeFileDir = os.path.dirname(localLargeFile)
1033                 if not os.path.isdir(largeFileDir):
1034                     os.makedirs(largeFileDir)
1035                 shutil.move(contentTempFile, localLargeFile)
1036                 self.addLargeFile(relPath)
1037                 if gitConfigBool('git-p4.largeFilePush'):
1038                     self.pushFile(localLargeFile)
1039                 if verbose:
1040                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1041         return (git_mode, contents)
1042
1043 class MockLFS(LargeFileSystem):
1044     """Mock large file system for testing."""
1045
1046     def generatePointer(self, contentFile):
1047         """The pointer content is the original content prefixed with "pointer-".
1048            The local filename of the large file storage is derived from the file content.
1049            """
1050         with open(contentFile, 'r') as f:
1051             content = next(f)
1052             gitMode = '100644'
1053             pointerContents = 'pointer-' + content
1054             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1055             return (gitMode, pointerContents, localLargeFile)
1056
1057     def pushFile(self, localLargeFile):
1058         """The remote filename of the large file storage is the same as the local
1059            one but in a different directory.
1060            """
1061         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1062         if not os.path.exists(remotePath):
1063             os.makedirs(remotePath)
1064         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1065
1066 class GitLFS(LargeFileSystem):
1067     """Git LFS as backend for the git-p4 large file system.
1068        See https://git-lfs.github.com/ for details."""
1069
1070     def __init__(self, *args):
1071         LargeFileSystem.__init__(self, *args)
1072         self.baseGitAttributes = []
1073
1074     def generatePointer(self, contentFile):
1075         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1076            mode and content which is stored in the Git repository instead of
1077            the actual content. Return also the new location of the actual
1078            content.
1079            """
1080         if os.path.getsize(contentFile) == 0:
1081             return (None, '', None)
1082
1083         pointerProcess = subprocess.Popen(
1084             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1085             stdout=subprocess.PIPE
1086         )
1087         pointerFile = pointerProcess.stdout.read()
1088         if pointerProcess.wait():
1089             os.remove(contentFile)
1090             die('git-lfs pointer command failed. Did you install the extension?')
1091
1092         # Git LFS removed the preamble in the output of the 'pointer' command
1093         # starting from version 1.2.0. Check for the preamble here to support
1094         # earlier versions.
1095         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1096         if pointerFile.startswith('Git LFS pointer for'):
1097             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1098
1099         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1100         localLargeFile = os.path.join(
1101             os.getcwd(),
1102             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1103             oid,
1104         )
1105         # LFS Spec states that pointer files should not have the executable bit set.
1106         gitMode = '100644'
1107         return (gitMode, pointerFile, localLargeFile)
1108
1109     def pushFile(self, localLargeFile):
1110         uploadProcess = subprocess.Popen(
1111             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1112         )
1113         if uploadProcess.wait():
1114             die('git-lfs push command failed. Did you define a remote?')
1115
1116     def generateGitAttributes(self):
1117         return (
1118             self.baseGitAttributes +
1119             [
1120                 '\n',
1121                 '#\n',
1122                 '# Git LFS (see https://git-lfs.github.com/)\n',
1123                 '#\n',
1124             ] +
1125             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1126                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1127             ] +
1128             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1129                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1130             ]
1131         )
1132
1133     def addLargeFile(self, relPath):
1134         LargeFileSystem.addLargeFile(self, relPath)
1135         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1136
1137     def removeLargeFile(self, relPath):
1138         LargeFileSystem.removeLargeFile(self, relPath)
1139         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1140
1141     def processContent(self, git_mode, relPath, contents):
1142         if relPath == '.gitattributes':
1143             self.baseGitAttributes = contents
1144             return (git_mode, self.generateGitAttributes())
1145         else:
1146             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1147
1148 class Command:
1149     def __init__(self):
1150         self.usage = "usage: %prog [options]"
1151         self.needsGit = True
1152         self.verbose = False
1153
1154 class P4UserMap:
1155     def __init__(self):
1156         self.userMapFromPerforceServer = False
1157         self.myP4UserId = None
1158
1159     def p4UserId(self):
1160         if self.myP4UserId:
1161             return self.myP4UserId
1162
1163         results = p4CmdList("user -o")
1164         for r in results:
1165             if r.has_key('User'):
1166                 self.myP4UserId = r['User']
1167                 return r['User']
1168         die("Could not find your p4 user id")
1169
1170     def p4UserIsMe(self, p4User):
1171         # return True if the given p4 user is actually me
1172         me = self.p4UserId()
1173         if not p4User or p4User != me:
1174             return False
1175         else:
1176             return True
1177
1178     def getUserCacheFilename(self):
1179         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1180         return home + "/.gitp4-usercache.txt"
1181
1182     def getUserMapFromPerforceServer(self):
1183         if self.userMapFromPerforceServer:
1184             return
1185         self.users = {}
1186         self.emails = {}
1187
1188         for output in p4CmdList("users"):
1189             if not output.has_key("User"):
1190                 continue
1191             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1192             self.emails[output["Email"]] = output["User"]
1193
1194         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1195         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1196             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1197             if mapUser and len(mapUser[0]) == 3:
1198                 user = mapUser[0][0]
1199                 fullname = mapUser[0][1]
1200                 email = mapUser[0][2]
1201                 self.users[user] = fullname + " <" + email + ">"
1202                 self.emails[email] = user
1203
1204         s = ''
1205         for (key, val) in self.users.items():
1206             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1207
1208         open(self.getUserCacheFilename(), "wb").write(s)
1209         self.userMapFromPerforceServer = True
1210
1211     def loadUserMapFromCache(self):
1212         self.users = {}
1213         self.userMapFromPerforceServer = False
1214         try:
1215             cache = open(self.getUserCacheFilename(), "rb")
1216             lines = cache.readlines()
1217             cache.close()
1218             for line in lines:
1219                 entry = line.strip().split("\t")
1220                 self.users[entry[0]] = entry[1]
1221         except IOError:
1222             self.getUserMapFromPerforceServer()
1223
1224 class P4Debug(Command):
1225     def __init__(self):
1226         Command.__init__(self)
1227         self.options = []
1228         self.description = "A tool to debug the output of p4 -G."
1229         self.needsGit = False
1230
1231     def run(self, args):
1232         j = 0
1233         for output in p4CmdList(args):
1234             print 'Element: %d' % j
1235             j += 1
1236             print output
1237         return True
1238
1239 class P4RollBack(Command):
1240     def __init__(self):
1241         Command.__init__(self)
1242         self.options = [
1243             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1244         ]
1245         self.description = "A tool to debug the multi-branch import. Don't use :)"
1246         self.rollbackLocalBranches = False
1247
1248     def run(self, args):
1249         if len(args) != 1:
1250             return False
1251         maxChange = int(args[0])
1252
1253         if "p4ExitCode" in p4Cmd("changes -m 1"):
1254             die("Problems executing p4");
1255
1256         if self.rollbackLocalBranches:
1257             refPrefix = "refs/heads/"
1258             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1259         else:
1260             refPrefix = "refs/remotes/"
1261             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1262
1263         for line in lines:
1264             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1265                 line = line.strip()
1266                 ref = refPrefix + line
1267                 log = extractLogMessageFromGitCommit(ref)
1268                 settings = extractSettingsGitLog(log)
1269
1270                 depotPaths = settings['depot-paths']
1271                 change = settings['change']
1272
1273                 changed = False
1274
1275                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1276                                                            for p in depotPaths]))) == 0:
1277                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1278                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1279                     continue
1280
1281                 while change and int(change) > maxChange:
1282                     changed = True
1283                     if self.verbose:
1284                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1285                     system("git update-ref %s \"%s^\"" % (ref, ref))
1286                     log = extractLogMessageFromGitCommit(ref)
1287                     settings =  extractSettingsGitLog(log)
1288
1289
1290                     depotPaths = settings['depot-paths']
1291                     change = settings['change']
1292
1293                 if changed:
1294                     print "%s rewound to %s" % (ref, change)
1295
1296         return True
1297
1298 class P4Submit(Command, P4UserMap):
1299
1300     conflict_behavior_choices = ("ask", "skip", "quit")
1301
1302     def __init__(self):
1303         Command.__init__(self)
1304         P4UserMap.__init__(self)
1305         self.options = [
1306                 optparse.make_option("--origin", dest="origin"),
1307                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1308                 # preserve the user, requires relevant p4 permissions
1309                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1310                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1311                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1312                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1313                 optparse.make_option("--conflict", dest="conflict_behavior",
1314                                      choices=self.conflict_behavior_choices),
1315                 optparse.make_option("--branch", dest="branch"),
1316                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1317                                      help="Shelve instead of submit. Shelved files are reverted, "
1318                                      "restoring the workspace to the state before the shelve"),
1319                 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1320                                      metavar="CHANGELIST",
1321                                      help="update an existing shelved changelist, implies --shelve")
1322         ]
1323         self.description = "Submit changes from git to the perforce depot."
1324         self.usage += " [name of git branch to submit into perforce depot]"
1325         self.origin = ""
1326         self.detectRenames = False
1327         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1328         self.dry_run = False
1329         self.shelve = False
1330         self.update_shelve = None
1331         self.prepare_p4_only = False
1332         self.conflict_behavior = None
1333         self.isWindows = (platform.system() == "Windows")
1334         self.exportLabels = False
1335         self.p4HasMoveCommand = p4_has_move_command()
1336         self.branch = None
1337
1338         if gitConfig('git-p4.largeFileSystem'):
1339             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1340
1341     def check(self):
1342         if len(p4CmdList("opened ...")) > 0:
1343             die("You have files opened with perforce! Close them before starting the sync.")
1344
1345     def separate_jobs_from_description(self, message):
1346         """Extract and return a possible Jobs field in the commit
1347            message.  It goes into a separate section in the p4 change
1348            specification.
1349
1350            A jobs line starts with "Jobs:" and looks like a new field
1351            in a form.  Values are white-space separated on the same
1352            line or on following lines that start with a tab.
1353
1354            This does not parse and extract the full git commit message
1355            like a p4 form.  It just sees the Jobs: line as a marker
1356            to pass everything from then on directly into the p4 form,
1357            but outside the description section.
1358
1359            Return a tuple (stripped log message, jobs string)."""
1360
1361         m = re.search(r'^Jobs:', message, re.MULTILINE)
1362         if m is None:
1363             return (message, None)
1364
1365         jobtext = message[m.start():]
1366         stripped_message = message[:m.start()].rstrip()
1367         return (stripped_message, jobtext)
1368
1369     def prepareLogMessage(self, template, message, jobs):
1370         """Edits the template returned from "p4 change -o" to insert
1371            the message in the Description field, and the jobs text in
1372            the Jobs field."""
1373         result = ""
1374
1375         inDescriptionSection = False
1376
1377         for line in template.split("\n"):
1378             if line.startswith("#"):
1379                 result += line + "\n"
1380                 continue
1381
1382             if inDescriptionSection:
1383                 if line.startswith("Files:") or line.startswith("Jobs:"):
1384                     inDescriptionSection = False
1385                     # insert Jobs section
1386                     if jobs:
1387                         result += jobs + "\n"
1388                 else:
1389                     continue
1390             else:
1391                 if line.startswith("Description:"):
1392                     inDescriptionSection = True
1393                     line += "\n"
1394                     for messageLine in message.split("\n"):
1395                         line += "\t" + messageLine + "\n"
1396
1397             result += line + "\n"
1398
1399         return result
1400
1401     def patchRCSKeywords(self, file, pattern):
1402         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1403         (handle, outFileName) = tempfile.mkstemp(dir='.')
1404         try:
1405             outFile = os.fdopen(handle, "w+")
1406             inFile = open(file, "r")
1407             regexp = re.compile(pattern, re.VERBOSE)
1408             for line in inFile.readlines():
1409                 line = regexp.sub(r'$\1$', line)
1410                 outFile.write(line)
1411             inFile.close()
1412             outFile.close()
1413             # Forcibly overwrite the original file
1414             os.unlink(file)
1415             shutil.move(outFileName, file)
1416         except:
1417             # cleanup our temporary file
1418             os.unlink(outFileName)
1419             print "Failed to strip RCS keywords in %s" % file
1420             raise
1421
1422         print "Patched up RCS keywords in %s" % file
1423
1424     def p4UserForCommit(self,id):
1425         # Return the tuple (perforce user,git email) for a given git commit id
1426         self.getUserMapFromPerforceServer()
1427         gitEmail = read_pipe(["git", "log", "--max-count=1",
1428                               "--format=%ae", id])
1429         gitEmail = gitEmail.strip()
1430         if not self.emails.has_key(gitEmail):
1431             return (None,gitEmail)
1432         else:
1433             return (self.emails[gitEmail],gitEmail)
1434
1435     def checkValidP4Users(self,commits):
1436         # check if any git authors cannot be mapped to p4 users
1437         for id in commits:
1438             (user,email) = self.p4UserForCommit(id)
1439             if not user:
1440                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1441                 if gitConfigBool("git-p4.allowMissingP4Users"):
1442                     print "%s" % msg
1443                 else:
1444                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1445
1446     def lastP4Changelist(self):
1447         # Get back the last changelist number submitted in this client spec. This
1448         # then gets used to patch up the username in the change. If the same
1449         # client spec is being used by multiple processes then this might go
1450         # wrong.
1451         results = p4CmdList("client -o")        # find the current client
1452         client = None
1453         for r in results:
1454             if r.has_key('Client'):
1455                 client = r['Client']
1456                 break
1457         if not client:
1458             die("could not get client spec")
1459         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1460         for r in results:
1461             if r.has_key('change'):
1462                 return r['change']
1463         die("Could not get changelist number for last submit - cannot patch up user details")
1464
1465     def modifyChangelistUser(self, changelist, newUser):
1466         # fixup the user field of a changelist after it has been submitted.
1467         changes = p4CmdList("change -o %s" % changelist)
1468         if len(changes) != 1:
1469             die("Bad output from p4 change modifying %s to user %s" %
1470                 (changelist, newUser))
1471
1472         c = changes[0]
1473         if c['User'] == newUser: return   # nothing to do
1474         c['User'] = newUser
1475         input = marshal.dumps(c)
1476
1477         result = p4CmdList("change -f -i", stdin=input)
1478         for r in result:
1479             if r.has_key('code'):
1480                 if r['code'] == 'error':
1481                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1482             if r.has_key('data'):
1483                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1484                 return
1485         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1486
1487     def canChangeChangelists(self):
1488         # check to see if we have p4 admin or super-user permissions, either of
1489         # which are required to modify changelists.
1490         results = p4CmdList(["protects", self.depotPath])
1491         for r in results:
1492             if r.has_key('perm'):
1493                 if r['perm'] == 'admin':
1494                     return 1
1495                 if r['perm'] == 'super':
1496                     return 1
1497         return 0
1498
1499     def prepareSubmitTemplate(self, changelist=None):
1500         """Run "p4 change -o" to grab a change specification template.
1501            This does not use "p4 -G", as it is nice to keep the submission
1502            template in original order, since a human might edit it.
1503
1504            Remove lines in the Files section that show changes to files
1505            outside the depot path we're committing into."""
1506
1507         [upstream, settings] = findUpstreamBranchPoint()
1508
1509         template = ""
1510         inFilesSection = False
1511         args = ['change', '-o']
1512         if changelist:
1513             args.append(str(changelist))
1514
1515         for line in p4_read_pipe_lines(args):
1516             if line.endswith("\r\n"):
1517                 line = line[:-2] + "\n"
1518             if inFilesSection:
1519                 if line.startswith("\t"):
1520                     # path starts and ends with a tab
1521                     path = line[1:]
1522                     lastTab = path.rfind("\t")
1523                     if lastTab != -1:
1524                         path = path[:lastTab]
1525                         if settings.has_key('depot-paths'):
1526                             if not [p for p in settings['depot-paths']
1527                                     if p4PathStartsWith(path, p)]:
1528                                 continue
1529                         else:
1530                             if not p4PathStartsWith(path, self.depotPath):
1531                                 continue
1532                 else:
1533                     inFilesSection = False
1534             else:
1535                 if line.startswith("Files:"):
1536                     inFilesSection = True
1537
1538             template += line
1539
1540         return template
1541
1542     def edit_template(self, template_file):
1543         """Invoke the editor to let the user change the submission
1544            message.  Return true if okay to continue with the submit."""
1545
1546         # if configured to skip the editing part, just submit
1547         if gitConfigBool("git-p4.skipSubmitEdit"):
1548             return True
1549
1550         # look at the modification time, to check later if the user saved
1551         # the file
1552         mtime = os.stat(template_file).st_mtime
1553
1554         # invoke the editor
1555         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1556             editor = os.environ.get("P4EDITOR")
1557         else:
1558             editor = read_pipe("git var GIT_EDITOR").strip()
1559         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1560
1561         # If the file was not saved, prompt to see if this patch should
1562         # be skipped.  But skip this verification step if configured so.
1563         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1564             return True
1565
1566         # modification time updated means user saved the file
1567         if os.stat(template_file).st_mtime > mtime:
1568             return True
1569
1570         while True:
1571             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1572             if response == 'y':
1573                 return True
1574             if response == 'n':
1575                 return False
1576
1577     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1578         # diff
1579         if os.environ.has_key("P4DIFF"):
1580             del(os.environ["P4DIFF"])
1581         diff = ""
1582         for editedFile in editedFiles:
1583             diff += p4_read_pipe(['diff', '-du',
1584                                   wildcard_encode(editedFile)])
1585
1586         # new file diff
1587         newdiff = ""
1588         for newFile in filesToAdd:
1589             newdiff += "==== new file ====\n"
1590             newdiff += "--- /dev/null\n"
1591             newdiff += "+++ %s\n" % newFile
1592
1593             is_link = os.path.islink(newFile)
1594             expect_link = newFile in symlinks
1595
1596             if is_link and expect_link:
1597                 newdiff += "+%s\n" % os.readlink(newFile)
1598             else:
1599                 f = open(newFile, "r")
1600                 for line in f.readlines():
1601                     newdiff += "+" + line
1602                 f.close()
1603
1604         return (diff + newdiff).replace('\r\n', '\n')
1605
1606     def applyCommit(self, id):
1607         """Apply one commit, return True if it succeeded."""
1608
1609         print "Applying", read_pipe(["git", "show", "-s",
1610                                      "--format=format:%h %s", id])
1611
1612         (p4User, gitEmail) = self.p4UserForCommit(id)
1613
1614         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1615         filesToAdd = set()
1616         filesToChangeType = set()
1617         filesToDelete = set()
1618         editedFiles = set()
1619         pureRenameCopy = set()
1620         symlinks = set()
1621         filesToChangeExecBit = {}
1622         all_files = list()
1623
1624         for line in diff:
1625             diff = parseDiffTreeEntry(line)
1626             modifier = diff['status']
1627             path = diff['src']
1628             all_files.append(path)
1629
1630             if modifier == "M":
1631                 p4_edit(path)
1632                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1633                     filesToChangeExecBit[path] = diff['dst_mode']
1634                 editedFiles.add(path)
1635             elif modifier == "A":
1636                 filesToAdd.add(path)
1637                 filesToChangeExecBit[path] = diff['dst_mode']
1638                 if path in filesToDelete:
1639                     filesToDelete.remove(path)
1640
1641                 dst_mode = int(diff['dst_mode'], 8)
1642                 if dst_mode == 0120000:
1643                     symlinks.add(path)
1644
1645             elif modifier == "D":
1646                 filesToDelete.add(path)
1647                 if path in filesToAdd:
1648                     filesToAdd.remove(path)
1649             elif modifier == "C":
1650                 src, dest = diff['src'], diff['dst']
1651                 p4_integrate(src, dest)
1652                 pureRenameCopy.add(dest)
1653                 if diff['src_sha1'] != diff['dst_sha1']:
1654                     p4_edit(dest)
1655                     pureRenameCopy.discard(dest)
1656                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1657                     p4_edit(dest)
1658                     pureRenameCopy.discard(dest)
1659                     filesToChangeExecBit[dest] = diff['dst_mode']
1660                 if self.isWindows:
1661                     # turn off read-only attribute
1662                     os.chmod(dest, stat.S_IWRITE)
1663                 os.unlink(dest)
1664                 editedFiles.add(dest)
1665             elif modifier == "R":
1666                 src, dest = diff['src'], diff['dst']
1667                 if self.p4HasMoveCommand:
1668                     p4_edit(src)        # src must be open before move
1669                     p4_move(src, dest)  # opens for (move/delete, move/add)
1670                 else:
1671                     p4_integrate(src, dest)
1672                     if diff['src_sha1'] != diff['dst_sha1']:
1673                         p4_edit(dest)
1674                     else:
1675                         pureRenameCopy.add(dest)
1676                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1677                     if not self.p4HasMoveCommand:
1678                         p4_edit(dest)   # with move: already open, writable
1679                     filesToChangeExecBit[dest] = diff['dst_mode']
1680                 if not self.p4HasMoveCommand:
1681                     if self.isWindows:
1682                         os.chmod(dest, stat.S_IWRITE)
1683                     os.unlink(dest)
1684                     filesToDelete.add(src)
1685                 editedFiles.add(dest)
1686             elif modifier == "T":
1687                 filesToChangeType.add(path)
1688             else:
1689                 die("unknown modifier %s for %s" % (modifier, path))
1690
1691         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1692         patchcmd = diffcmd + " | git apply "
1693         tryPatchCmd = patchcmd + "--check -"
1694         applyPatchCmd = patchcmd + "--check --apply -"
1695         patch_succeeded = True
1696
1697         if os.system(tryPatchCmd) != 0:
1698             fixed_rcs_keywords = False
1699             patch_succeeded = False
1700             print "Unfortunately applying the change failed!"
1701
1702             # Patch failed, maybe it's just RCS keyword woes. Look through
1703             # the patch to see if that's possible.
1704             if gitConfigBool("git-p4.attemptRCSCleanup"):
1705                 file = None
1706                 pattern = None
1707                 kwfiles = {}
1708                 for file in editedFiles | filesToDelete:
1709                     # did this file's delta contain RCS keywords?
1710                     pattern = p4_keywords_regexp_for_file(file)
1711
1712                     if pattern:
1713                         # this file is a possibility...look for RCS keywords.
1714                         regexp = re.compile(pattern, re.VERBOSE)
1715                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1716                             if regexp.search(line):
1717                                 if verbose:
1718                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1719                                 kwfiles[file] = pattern
1720                                 break
1721
1722                 for file in kwfiles:
1723                     if verbose:
1724                         print "zapping %s with %s" % (line,pattern)
1725                     # File is being deleted, so not open in p4.  Must
1726                     # disable the read-only bit on windows.
1727                     if self.isWindows and file not in editedFiles:
1728                         os.chmod(file, stat.S_IWRITE)
1729                     self.patchRCSKeywords(file, kwfiles[file])
1730                     fixed_rcs_keywords = True
1731
1732             if fixed_rcs_keywords:
1733                 print "Retrying the patch with RCS keywords cleaned up"
1734                 if os.system(tryPatchCmd) == 0:
1735                     patch_succeeded = True
1736
1737         if not patch_succeeded:
1738             for f in editedFiles:
1739                 p4_revert(f)
1740             return False
1741
1742         #
1743         # Apply the patch for real, and do add/delete/+x handling.
1744         #
1745         system(applyPatchCmd)
1746
1747         for f in filesToChangeType:
1748             p4_edit(f, "-t", "auto")
1749         for f in filesToAdd:
1750             p4_add(f)
1751         for f in filesToDelete:
1752             p4_revert(f)
1753             p4_delete(f)
1754
1755         # Set/clear executable bits
1756         for f in filesToChangeExecBit.keys():
1757             mode = filesToChangeExecBit[f]
1758             setP4ExecBit(f, mode)
1759
1760         if self.update_shelve:
1761             print("all_files = %s" % str(all_files))
1762             p4_reopen_in_change(self.update_shelve, all_files)
1763
1764         #
1765         # Build p4 change description, starting with the contents
1766         # of the git commit message.
1767         #
1768         logMessage = extractLogMessageFromGitCommit(id)
1769         logMessage = logMessage.strip()
1770         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1771
1772         template = self.prepareSubmitTemplate(self.update_shelve)
1773         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1774
1775         if self.preserveUser:
1776            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1777
1778         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1779             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1780             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1781             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1782
1783         separatorLine = "######## everything below this line is just the diff #######\n"
1784         if not self.prepare_p4_only:
1785             submitTemplate += separatorLine
1786             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1787
1788         (handle, fileName) = tempfile.mkstemp()
1789         tmpFile = os.fdopen(handle, "w+b")
1790         if self.isWindows:
1791             submitTemplate = submitTemplate.replace("\n", "\r\n")
1792         tmpFile.write(submitTemplate)
1793         tmpFile.close()
1794
1795         if self.prepare_p4_only:
1796             #
1797             # Leave the p4 tree prepared, and the submit template around
1798             # and let the user decide what to do next
1799             #
1800             print
1801             print "P4 workspace prepared for submission."
1802             print "To submit or revert, go to client workspace"
1803             print "  " + self.clientPath
1804             print
1805             print "To submit, use \"p4 submit\" to write a new description,"
1806             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1807                   " \"git p4\"." % fileName
1808             print "You can delete the file \"%s\" when finished." % fileName
1809
1810             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1811                 print "To preserve change ownership by user %s, you must\n" \
1812                       "do \"p4 change -f <change>\" after submitting and\n" \
1813                       "edit the User field."
1814             if pureRenameCopy:
1815                 print "After submitting, renamed files must be re-synced."
1816                 print "Invoke \"p4 sync -f\" on each of these files:"
1817                 for f in pureRenameCopy:
1818                     print "  " + f
1819
1820             print
1821             print "To revert the changes, use \"p4 revert ...\", and delete"
1822             print "the submit template file \"%s\"" % fileName
1823             if filesToAdd:
1824                 print "Since the commit adds new files, they must be deleted:"
1825                 for f in filesToAdd:
1826                     print "  " + f
1827             print
1828             return True
1829
1830         #
1831         # Let the user edit the change description, then submit it.
1832         #
1833         submitted = False
1834
1835         try:
1836             if self.edit_template(fileName):
1837                 # read the edited message and submit
1838                 tmpFile = open(fileName, "rb")
1839                 message = tmpFile.read()
1840                 tmpFile.close()
1841                 if self.isWindows:
1842                     message = message.replace("\r\n", "\n")
1843                 submitTemplate = message[:message.index(separatorLine)]
1844
1845                 if self.update_shelve:
1846                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1847                 elif self.shelve:
1848                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1849                 else:
1850                     p4_write_pipe(['submit', '-i'], submitTemplate)
1851                     # The rename/copy happened by applying a patch that created a
1852                     # new file.  This leaves it writable, which confuses p4.
1853                     for f in pureRenameCopy:
1854                         p4_sync(f, "-f")
1855
1856                 if self.preserveUser:
1857                     if p4User:
1858                         # Get last changelist number. Cannot easily get it from
1859                         # the submit command output as the output is
1860                         # unmarshalled.
1861                         changelist = self.lastP4Changelist()
1862                         self.modifyChangelistUser(changelist, p4User)
1863
1864                 submitted = True
1865
1866         finally:
1867             # skip this patch
1868             if not submitted or self.shelve:
1869                 if self.shelve:
1870                     print ("Reverting shelved files.")
1871                 else:
1872                     print ("Submission cancelled, undoing p4 changes.")
1873                 for f in editedFiles | filesToDelete:
1874                     p4_revert(f)
1875                 for f in filesToAdd:
1876                     p4_revert(f)
1877                     os.remove(f)
1878
1879         os.remove(fileName)
1880         return submitted
1881
1882     # Export git tags as p4 labels. Create a p4 label and then tag
1883     # with that.
1884     def exportGitTags(self, gitTags):
1885         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1886         if len(validLabelRegexp) == 0:
1887             validLabelRegexp = defaultLabelRegexp
1888         m = re.compile(validLabelRegexp)
1889
1890         for name in gitTags:
1891
1892             if not m.match(name):
1893                 if verbose:
1894                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1895                 continue
1896
1897             # Get the p4 commit this corresponds to
1898             logMessage = extractLogMessageFromGitCommit(name)
1899             values = extractSettingsGitLog(logMessage)
1900
1901             if not values.has_key('change'):
1902                 # a tag pointing to something not sent to p4; ignore
1903                 if verbose:
1904                     print "git tag %s does not give a p4 commit" % name
1905                 continue
1906             else:
1907                 changelist = values['change']
1908
1909             # Get the tag details.
1910             inHeader = True
1911             isAnnotated = False
1912             body = []
1913             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1914                 l = l.strip()
1915                 if inHeader:
1916                     if re.match(r'tag\s+', l):
1917                         isAnnotated = True
1918                     elif re.match(r'\s*$', l):
1919                         inHeader = False
1920                         continue
1921                 else:
1922                     body.append(l)
1923
1924             if not isAnnotated:
1925                 body = ["lightweight tag imported by git p4\n"]
1926
1927             # Create the label - use the same view as the client spec we are using
1928             clientSpec = getClientSpec()
1929
1930             labelTemplate  = "Label: %s\n" % name
1931             labelTemplate += "Description:\n"
1932             for b in body:
1933                 labelTemplate += "\t" + b + "\n"
1934             labelTemplate += "View:\n"
1935             for depot_side in clientSpec.mappings:
1936                 labelTemplate += "\t%s\n" % depot_side
1937
1938             if self.dry_run:
1939                 print "Would create p4 label %s for tag" % name
1940             elif self.prepare_p4_only:
1941                 print "Not creating p4 label %s for tag due to option" \
1942                       " --prepare-p4-only" % name
1943             else:
1944                 p4_write_pipe(["label", "-i"], labelTemplate)
1945
1946                 # Use the label
1947                 p4_system(["tag", "-l", name] +
1948                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1949
1950                 if verbose:
1951                     print "created p4 label for tag %s" % name
1952
1953     def run(self, args):
1954         if len(args) == 0:
1955             self.master = currentGitBranch()
1956         elif len(args) == 1:
1957             self.master = args[0]
1958             if not branchExists(self.master):
1959                 die("Branch %s does not exist" % self.master)
1960         else:
1961             return False
1962
1963         if self.master:
1964             allowSubmit = gitConfig("git-p4.allowSubmit")
1965             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1966                 die("%s is not in git-p4.allowSubmit" % self.master)
1967
1968         [upstream, settings] = findUpstreamBranchPoint()
1969         self.depotPath = settings['depot-paths'][0]
1970         if len(self.origin) == 0:
1971             self.origin = upstream
1972
1973         if self.update_shelve:
1974             self.shelve = True
1975
1976         if self.preserveUser:
1977             if not self.canChangeChangelists():
1978                 die("Cannot preserve user names without p4 super-user or admin permissions")
1979
1980         # if not set from the command line, try the config file
1981         if self.conflict_behavior is None:
1982             val = gitConfig("git-p4.conflict")
1983             if val:
1984                 if val not in self.conflict_behavior_choices:
1985                     die("Invalid value '%s' for config git-p4.conflict" % val)
1986             else:
1987                 val = "ask"
1988             self.conflict_behavior = val
1989
1990         if self.verbose:
1991             print "Origin branch is " + self.origin
1992
1993         if len(self.depotPath) == 0:
1994             print "Internal error: cannot locate perforce depot path from existing branches"
1995             sys.exit(128)
1996
1997         self.useClientSpec = False
1998         if gitConfigBool("git-p4.useclientspec"):
1999             self.useClientSpec = True
2000         if self.useClientSpec:
2001             self.clientSpecDirs = getClientSpec()
2002
2003         # Check for the existence of P4 branches
2004         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2005
2006         if self.useClientSpec and not branchesDetected:
2007             # all files are relative to the client spec
2008             self.clientPath = getClientRoot()
2009         else:
2010             self.clientPath = p4Where(self.depotPath)
2011
2012         if self.clientPath == "":
2013             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2014
2015         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2016         self.oldWorkingDirectory = os.getcwd()
2017
2018         # ensure the clientPath exists
2019         new_client_dir = False
2020         if not os.path.exists(self.clientPath):
2021             new_client_dir = True
2022             os.makedirs(self.clientPath)
2023
2024         chdir(self.clientPath, is_client_path=True)
2025         if self.dry_run:
2026             print "Would synchronize p4 checkout in %s" % self.clientPath
2027         else:
2028             print "Synchronizing p4 checkout..."
2029             if new_client_dir:
2030                 # old one was destroyed, and maybe nobody told p4
2031                 p4_sync("...", "-f")
2032             else:
2033                 p4_sync("...")
2034         self.check()
2035
2036         commits = []
2037         if self.master:
2038             commitish = self.master
2039         else:
2040             commitish = 'HEAD'
2041
2042         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2043             commits.append(line.strip())
2044         commits.reverse()
2045
2046         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2047             self.checkAuthorship = False
2048         else:
2049             self.checkAuthorship = True
2050
2051         if self.preserveUser:
2052             self.checkValidP4Users(commits)
2053
2054         #
2055         # Build up a set of options to be passed to diff when
2056         # submitting each commit to p4.
2057         #
2058         if self.detectRenames:
2059             # command-line -M arg
2060             self.diffOpts = "-M"
2061         else:
2062             # If not explicitly set check the config variable
2063             detectRenames = gitConfig("git-p4.detectRenames")
2064
2065             if detectRenames.lower() == "false" or detectRenames == "":
2066                 self.diffOpts = ""
2067             elif detectRenames.lower() == "true":
2068                 self.diffOpts = "-M"
2069             else:
2070                 self.diffOpts = "-M%s" % detectRenames
2071
2072         # no command-line arg for -C or --find-copies-harder, just
2073         # config variables
2074         detectCopies = gitConfig("git-p4.detectCopies")
2075         if detectCopies.lower() == "false" or detectCopies == "":
2076             pass
2077         elif detectCopies.lower() == "true":
2078             self.diffOpts += " -C"
2079         else:
2080             self.diffOpts += " -C%s" % detectCopies
2081
2082         if gitConfigBool("git-p4.detectCopiesHarder"):
2083             self.diffOpts += " --find-copies-harder"
2084
2085         #
2086         # Apply the commits, one at a time.  On failure, ask if should
2087         # continue to try the rest of the patches, or quit.
2088         #
2089         if self.dry_run:
2090             print "Would apply"
2091         applied = []
2092         last = len(commits) - 1
2093         for i, commit in enumerate(commits):
2094             if self.dry_run:
2095                 print " ", read_pipe(["git", "show", "-s",
2096                                       "--format=format:%h %s", commit])
2097                 ok = True
2098             else:
2099                 ok = self.applyCommit(commit)
2100             if ok:
2101                 applied.append(commit)
2102             else:
2103                 if self.prepare_p4_only and i < last:
2104                     print "Processing only the first commit due to option" \
2105                           " --prepare-p4-only"
2106                     break
2107                 if i < last:
2108                     quit = False
2109                     while True:
2110                         # prompt for what to do, or use the option/variable
2111                         if self.conflict_behavior == "ask":
2112                             print "What do you want to do?"
2113                             response = raw_input("[s]kip this commit but apply"
2114                                                  " the rest, or [q]uit? ")
2115                             if not response:
2116                                 continue
2117                         elif self.conflict_behavior == "skip":
2118                             response = "s"
2119                         elif self.conflict_behavior == "quit":
2120                             response = "q"
2121                         else:
2122                             die("Unknown conflict_behavior '%s'" %
2123                                 self.conflict_behavior)
2124
2125                         if response[0] == "s":
2126                             print "Skipping this commit, but applying the rest"
2127                             break
2128                         if response[0] == "q":
2129                             print "Quitting"
2130                             quit = True
2131                             break
2132                     if quit:
2133                         break
2134
2135         chdir(self.oldWorkingDirectory)
2136         shelved_applied = "shelved" if self.shelve else "applied"
2137         if self.dry_run:
2138             pass
2139         elif self.prepare_p4_only:
2140             pass
2141         elif len(commits) == len(applied):
2142             print ("All commits {0}!".format(shelved_applied))
2143
2144             sync = P4Sync()
2145             if self.branch:
2146                 sync.branch = self.branch
2147             sync.run([])
2148
2149             rebase = P4Rebase()
2150             rebase.rebase()
2151
2152         else:
2153             if len(applied) == 0:
2154                 print ("No commits {0}.".format(shelved_applied))
2155             else:
2156                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2157                 for c in commits:
2158                     if c in applied:
2159                         star = "*"
2160                     else:
2161                         star = " "
2162                     print star, read_pipe(["git", "show", "-s",
2163                                            "--format=format:%h %s",  c])
2164                 print "You will have to do 'git p4 sync' and rebase."
2165
2166         if gitConfigBool("git-p4.exportLabels"):
2167             self.exportLabels = True
2168
2169         if self.exportLabels:
2170             p4Labels = getP4Labels(self.depotPath)
2171             gitTags = getGitTags()
2172
2173             missingGitTags = gitTags - p4Labels
2174             self.exportGitTags(missingGitTags)
2175
2176         # exit with error unless everything applied perfectly
2177         if len(commits) != len(applied):
2178                 sys.exit(1)
2179
2180         return True
2181
2182 class View(object):
2183     """Represent a p4 view ("p4 help views"), and map files in a
2184        repo according to the view."""
2185
2186     def __init__(self, client_name):
2187         self.mappings = []
2188         self.client_prefix = "//%s/" % client_name
2189         # cache results of "p4 where" to lookup client file locations
2190         self.client_spec_path_cache = {}
2191
2192     def append(self, view_line):
2193         """Parse a view line, splitting it into depot and client
2194            sides.  Append to self.mappings, preserving order.  This
2195            is only needed for tag creation."""
2196
2197         # Split the view line into exactly two words.  P4 enforces
2198         # structure on these lines that simplifies this quite a bit.
2199         #
2200         # Either or both words may be double-quoted.
2201         # Single quotes do not matter.
2202         # Double-quote marks cannot occur inside the words.
2203         # A + or - prefix is also inside the quotes.
2204         # There are no quotes unless they contain a space.
2205         # The line is already white-space stripped.
2206         # The two words are separated by a single space.
2207         #
2208         if view_line[0] == '"':
2209             # First word is double quoted.  Find its end.
2210             close_quote_index = view_line.find('"', 1)
2211             if close_quote_index <= 0:
2212                 die("No first-word closing quote found: %s" % view_line)
2213             depot_side = view_line[1:close_quote_index]
2214             # skip closing quote and space
2215             rhs_index = close_quote_index + 1 + 1
2216         else:
2217             space_index = view_line.find(" ")
2218             if space_index <= 0:
2219                 die("No word-splitting space found: %s" % view_line)
2220             depot_side = view_line[0:space_index]
2221             rhs_index = space_index + 1
2222
2223         # prefix + means overlay on previous mapping
2224         if depot_side.startswith("+"):
2225             depot_side = depot_side[1:]
2226
2227         # prefix - means exclude this path, leave out of mappings
2228         exclude = False
2229         if depot_side.startswith("-"):
2230             exclude = True
2231             depot_side = depot_side[1:]
2232
2233         if not exclude:
2234             self.mappings.append(depot_side)
2235
2236     def convert_client_path(self, clientFile):
2237         # chop off //client/ part to make it relative
2238         if not clientFile.startswith(self.client_prefix):
2239             die("No prefix '%s' on clientFile '%s'" %
2240                 (self.client_prefix, clientFile))
2241         return clientFile[len(self.client_prefix):]
2242
2243     def update_client_spec_path_cache(self, files):
2244         """ Caching file paths by "p4 where" batch query """
2245
2246         # List depot file paths exclude that already cached
2247         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2248
2249         if len(fileArgs) == 0:
2250             return  # All files in cache
2251
2252         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2253         for res in where_result:
2254             if "code" in res and res["code"] == "error":
2255                 # assume error is "... file(s) not in client view"
2256                 continue
2257             if "clientFile" not in res:
2258                 die("No clientFile in 'p4 where' output")
2259             if "unmap" in res:
2260                 # it will list all of them, but only one not unmap-ped
2261                 continue
2262             if gitConfigBool("core.ignorecase"):
2263                 res['depotFile'] = res['depotFile'].lower()
2264             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2265
2266         # not found files or unmap files set to ""
2267         for depotFile in fileArgs:
2268             if gitConfigBool("core.ignorecase"):
2269                 depotFile = depotFile.lower()
2270             if depotFile not in self.client_spec_path_cache:
2271                 self.client_spec_path_cache[depotFile] = ""
2272
2273     def map_in_client(self, depot_path):
2274         """Return the relative location in the client where this
2275            depot file should live.  Returns "" if the file should
2276            not be mapped in the client."""
2277
2278         if gitConfigBool("core.ignorecase"):
2279             depot_path = depot_path.lower()
2280
2281         if depot_path in self.client_spec_path_cache:
2282             return self.client_spec_path_cache[depot_path]
2283
2284         die( "Error: %s is not found in client spec path" % depot_path )
2285         return ""
2286
2287 class P4Sync(Command, P4UserMap):
2288     delete_actions = ( "delete", "move/delete", "purge" )
2289
2290     def __init__(self):
2291         Command.__init__(self)
2292         P4UserMap.__init__(self)
2293         self.options = [
2294                 optparse.make_option("--branch", dest="branch"),
2295                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2296                 optparse.make_option("--changesfile", dest="changesFile"),
2297                 optparse.make_option("--silent", dest="silent", action="store_true"),
2298                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2299                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2300                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2301                                      help="Import into refs/heads/ , not refs/remotes"),
2302                 optparse.make_option("--max-changes", dest="maxChanges",
2303                                      help="Maximum number of changes to import"),
2304                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2305                                      help="Internal block size to use when iteratively calling p4 changes"),
2306                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2307                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2308                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2309                                      help="Only sync files that are included in the Perforce Client Spec"),
2310                 optparse.make_option("-/", dest="cloneExclude",
2311                                      action="append", type="string",
2312                                      help="exclude depot path"),
2313         ]
2314         self.description = """Imports from Perforce into a git repository.\n
2315     example:
2316     //depot/my/project/ -- to import the current head
2317     //depot/my/project/@all -- to import everything
2318     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2319
2320     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2321
2322         self.usage += " //depot/path[@revRange]"
2323         self.silent = False
2324         self.createdBranches = set()
2325         self.committedChanges = set()
2326         self.branch = ""
2327         self.detectBranches = False
2328         self.detectLabels = False
2329         self.importLabels = False
2330         self.changesFile = ""
2331         self.syncWithOrigin = True
2332         self.importIntoRemotes = True
2333         self.maxChanges = ""
2334         self.changes_block_size = None
2335         self.keepRepoPath = False
2336         self.depotPaths = None
2337         self.p4BranchesInGit = []
2338         self.cloneExclude = []
2339         self.useClientSpec = False
2340         self.useClientSpec_from_options = False
2341         self.clientSpecDirs = None
2342         self.tempBranches = []
2343         self.tempBranchLocation = "refs/git-p4-tmp"
2344         self.largeFileSystem = None
2345
2346         if gitConfig('git-p4.largeFileSystem'):
2347             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2348             self.largeFileSystem = largeFileSystemConstructor(
2349                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2350             )
2351
2352         if gitConfig("git-p4.syncFromOrigin") == "false":
2353             self.syncWithOrigin = False
2354
2355     # This is required for the "append" cloneExclude action
2356     def ensure_value(self, attr, value):
2357         if not hasattr(self, attr) or getattr(self, attr) is None:
2358             setattr(self, attr, value)
2359         return getattr(self, attr)
2360
2361     # Force a checkpoint in fast-import and wait for it to finish
2362     def checkpoint(self):
2363         self.gitStream.write("checkpoint\n\n")
2364         self.gitStream.write("progress checkpoint\n\n")
2365         out = self.gitOutput.readline()
2366         if self.verbose:
2367             print "checkpoint finished: " + out
2368
2369     def extractFilesFromCommit(self, commit):
2370         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2371                              for path in self.cloneExclude]
2372         files = []
2373         fnum = 0
2374         while commit.has_key("depotFile%s" % fnum):
2375             path =  commit["depotFile%s" % fnum]
2376
2377             if [p for p in self.cloneExclude
2378                 if p4PathStartsWith(path, p)]:
2379                 found = False
2380             else:
2381                 found = [p for p in self.depotPaths
2382                          if p4PathStartsWith(path, p)]
2383             if not found:
2384                 fnum = fnum + 1
2385                 continue
2386
2387             file = {}
2388             file["path"] = path
2389             file["rev"] = commit["rev%s" % fnum]
2390             file["action"] = commit["action%s" % fnum]
2391             file["type"] = commit["type%s" % fnum]
2392             files.append(file)
2393             fnum = fnum + 1
2394         return files
2395
2396     def extractJobsFromCommit(self, commit):
2397         jobs = []
2398         jnum = 0
2399         while commit.has_key("job%s" % jnum):
2400             job = commit["job%s" % jnum]
2401             jobs.append(job)
2402             jnum = jnum + 1
2403         return jobs
2404
2405     def stripRepoPath(self, path, prefixes):
2406         """When streaming files, this is called to map a p4 depot path
2407            to where it should go in git.  The prefixes are either
2408            self.depotPaths, or self.branchPrefixes in the case of
2409            branch detection."""
2410
2411         if self.useClientSpec:
2412             # branch detection moves files up a level (the branch name)
2413             # from what client spec interpretation gives
2414             path = self.clientSpecDirs.map_in_client(path)
2415             if self.detectBranches:
2416                 for b in self.knownBranches:
2417                     if path.startswith(b + "/"):
2418                         path = path[len(b)+1:]
2419
2420         elif self.keepRepoPath:
2421             # Preserve everything in relative path name except leading
2422             # //depot/; just look at first prefix as they all should
2423             # be in the same depot.
2424             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2425             if p4PathStartsWith(path, depot):
2426                 path = path[len(depot):]
2427
2428         else:
2429             for p in prefixes:
2430                 if p4PathStartsWith(path, p):
2431                     path = path[len(p):]
2432                     break
2433
2434         path = wildcard_decode(path)
2435         return path
2436
2437     def splitFilesIntoBranches(self, commit):
2438         """Look at each depotFile in the commit to figure out to what
2439            branch it belongs."""
2440
2441         if self.clientSpecDirs:
2442             files = self.extractFilesFromCommit(commit)
2443             self.clientSpecDirs.update_client_spec_path_cache(files)
2444
2445         branches = {}
2446         fnum = 0
2447         while commit.has_key("depotFile%s" % fnum):
2448             path =  commit["depotFile%s" % fnum]
2449             found = [p for p in self.depotPaths
2450                      if p4PathStartsWith(path, p)]
2451             if not found:
2452                 fnum = fnum + 1
2453                 continue
2454
2455             file = {}
2456             file["path"] = path
2457             file["rev"] = commit["rev%s" % fnum]
2458             file["action"] = commit["action%s" % fnum]
2459             file["type"] = commit["type%s" % fnum]
2460             fnum = fnum + 1
2461
2462             # start with the full relative path where this file would
2463             # go in a p4 client
2464             if self.useClientSpec:
2465                 relPath = self.clientSpecDirs.map_in_client(path)
2466             else:
2467                 relPath = self.stripRepoPath(path, self.depotPaths)
2468
2469             for branch in self.knownBranches.keys():
2470                 # add a trailing slash so that a commit into qt/4.2foo
2471                 # doesn't end up in qt/4.2, e.g.
2472                 if relPath.startswith(branch + "/"):
2473                     if branch not in branches:
2474                         branches[branch] = []
2475                     branches[branch].append(file)
2476                     break
2477
2478         return branches
2479
2480     def writeToGitStream(self, gitMode, relPath, contents):
2481         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2482         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2483         for d in contents:
2484             self.gitStream.write(d)
2485         self.gitStream.write('\n')
2486
2487     def encodeWithUTF8(self, path):
2488         try:
2489             path.decode('ascii')
2490         except:
2491             encoding = 'utf8'
2492             if gitConfig('git-p4.pathEncoding'):
2493                 encoding = gitConfig('git-p4.pathEncoding')
2494             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2495             if self.verbose:
2496                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2497         return path
2498
2499     # output one file from the P4 stream
2500     # - helper for streamP4Files
2501
2502     def streamOneP4File(self, file, contents):
2503         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2504         relPath = self.encodeWithUTF8(relPath)
2505         if verbose:
2506             size = int(self.stream_file['fileSize'])
2507             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2508             sys.stdout.flush()
2509
2510         (type_base, type_mods) = split_p4_type(file["type"])
2511
2512         git_mode = "100644"
2513         if "x" in type_mods:
2514             git_mode = "100755"
2515         if type_base == "symlink":
2516             git_mode = "120000"
2517             # p4 print on a symlink sometimes contains "target\n";
2518             # if it does, remove the newline
2519             data = ''.join(contents)
2520             if not data:
2521                 # Some version of p4 allowed creating a symlink that pointed
2522                 # to nothing.  This causes p4 errors when checking out such
2523                 # a change, and errors here too.  Work around it by ignoring
2524                 # the bad symlink; hopefully a future change fixes it.
2525                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2526                 return
2527             elif data[-1] == '\n':
2528                 contents = [data[:-1]]
2529             else:
2530                 contents = [data]
2531
2532         if type_base == "utf16":
2533             # p4 delivers different text in the python output to -G
2534             # than it does when using "print -o", or normal p4 client
2535             # operations.  utf16 is converted to ascii or utf8, perhaps.
2536             # But ascii text saved as -t utf16 is completely mangled.
2537             # Invoke print -o to get the real contents.
2538             #
2539             # On windows, the newlines will always be mangled by print, so put
2540             # them back too.  This is not needed to the cygwin windows version,
2541             # just the native "NT" type.
2542             #
2543             try:
2544                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2545             except Exception as e:
2546                 if 'Translation of file content failed' in str(e):
2547                     type_base = 'binary'
2548                 else:
2549                     raise e
2550             else:
2551                 if p4_version_string().find('/NT') >= 0:
2552                     text = text.replace('\r\n', '\n')
2553                 contents = [ text ]
2554
2555         if type_base == "apple":
2556             # Apple filetype files will be streamed as a concatenation of
2557             # its appledouble header and the contents.  This is useless
2558             # on both macs and non-macs.  If using "print -q -o xx", it
2559             # will create "xx" with the data, and "%xx" with the header.
2560             # This is also not very useful.
2561             #
2562             # Ideally, someday, this script can learn how to generate
2563             # appledouble files directly and import those to git, but
2564             # non-mac machines can never find a use for apple filetype.
2565             print "\nIgnoring apple filetype file %s" % file['depotFile']
2566             return
2567
2568         # Note that we do not try to de-mangle keywords on utf16 files,
2569         # even though in theory somebody may want that.
2570         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2571         if pattern:
2572             regexp = re.compile(pattern, re.VERBOSE)
2573             text = ''.join(contents)
2574             text = regexp.sub(r'$\1$', text)
2575             contents = [ text ]
2576
2577         if self.largeFileSystem:
2578             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2579
2580         self.writeToGitStream(git_mode, relPath, contents)
2581
2582     def streamOneP4Deletion(self, file):
2583         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2584         relPath = self.encodeWithUTF8(relPath)
2585         if verbose:
2586             sys.stdout.write("delete %s\n" % relPath)
2587             sys.stdout.flush()
2588         self.gitStream.write("D %s\n" % relPath)
2589
2590         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2591             self.largeFileSystem.removeLargeFile(relPath)
2592
2593     # handle another chunk of streaming data
2594     def streamP4FilesCb(self, marshalled):
2595
2596         # catch p4 errors and complain
2597         err = None
2598         if "code" in marshalled:
2599             if marshalled["code"] == "error":
2600                 if "data" in marshalled:
2601                     err = marshalled["data"].rstrip()
2602
2603         if not err and 'fileSize' in self.stream_file:
2604             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2605             if required_bytes > 0:
2606                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2607                     os.getcwd(), required_bytes/1024/1024
2608                 )
2609
2610         if err:
2611             f = None
2612             if self.stream_have_file_info:
2613                 if "depotFile" in self.stream_file:
2614                     f = self.stream_file["depotFile"]
2615             # force a failure in fast-import, else an empty
2616             # commit will be made
2617             self.gitStream.write("\n")
2618             self.gitStream.write("die-now\n")
2619             self.gitStream.close()
2620             # ignore errors, but make sure it exits first
2621             self.importProcess.wait()
2622             if f:
2623                 die("Error from p4 print for %s: %s" % (f, err))
2624             else:
2625                 die("Error from p4 print: %s" % err)
2626
2627         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2628             # start of a new file - output the old one first
2629             self.streamOneP4File(self.stream_file, self.stream_contents)
2630             self.stream_file = {}
2631             self.stream_contents = []
2632             self.stream_have_file_info = False
2633
2634         # pick up the new file information... for the
2635         # 'data' field we need to append to our array
2636         for k in marshalled.keys():
2637             if k == 'data':
2638                 if 'streamContentSize' not in self.stream_file:
2639                     self.stream_file['streamContentSize'] = 0
2640                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2641                 self.stream_contents.append(marshalled['data'])
2642             else:
2643                 self.stream_file[k] = marshalled[k]
2644
2645         if (verbose and
2646             'streamContentSize' in self.stream_file and
2647             'fileSize' in self.stream_file and
2648             'depotFile' in self.stream_file):
2649             size = int(self.stream_file["fileSize"])
2650             if size > 0:
2651                 progress = 100*self.stream_file['streamContentSize']/size
2652                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2653                 sys.stdout.flush()
2654
2655         self.stream_have_file_info = True
2656
2657     # Stream directly from "p4 files" into "git fast-import"
2658     def streamP4Files(self, files):
2659         filesForCommit = []
2660         filesToRead = []
2661         filesToDelete = []
2662
2663         for f in files:
2664             filesForCommit.append(f)
2665             if f['action'] in self.delete_actions:
2666                 filesToDelete.append(f)
2667             else:
2668                 filesToRead.append(f)
2669
2670         # deleted files...
2671         for f in filesToDelete:
2672             self.streamOneP4Deletion(f)
2673
2674         if len(filesToRead) > 0:
2675             self.stream_file = {}
2676             self.stream_contents = []
2677             self.stream_have_file_info = False
2678
2679             # curry self argument
2680             def streamP4FilesCbSelf(entry):
2681                 self.streamP4FilesCb(entry)
2682
2683             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2684
2685             p4CmdList(["-x", "-", "print"],
2686                       stdin=fileArgs,
2687                       cb=streamP4FilesCbSelf)
2688
2689             # do the last chunk
2690             if self.stream_file.has_key('depotFile'):
2691                 self.streamOneP4File(self.stream_file, self.stream_contents)
2692
2693     def make_email(self, userid):
2694         if userid in self.users:
2695             return self.users[userid]
2696         else:
2697             return "%s <a@b>" % userid
2698
2699     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2700         """ Stream a p4 tag.
2701         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2702         """
2703
2704         if verbose:
2705             print "writing tag %s for commit %s" % (labelName, commit)
2706         gitStream.write("tag %s\n" % labelName)
2707         gitStream.write("from %s\n" % commit)
2708
2709         if labelDetails.has_key('Owner'):
2710             owner = labelDetails["Owner"]
2711         else:
2712             owner = None
2713
2714         # Try to use the owner of the p4 label, or failing that,
2715         # the current p4 user id.
2716         if owner:
2717             email = self.make_email(owner)
2718         else:
2719             email = self.make_email(self.p4UserId())
2720         tagger = "%s %s %s" % (email, epoch, self.tz)
2721
2722         gitStream.write("tagger %s\n" % tagger)
2723
2724         print "labelDetails=",labelDetails
2725         if labelDetails.has_key('Description'):
2726             description = labelDetails['Description']
2727         else:
2728             description = 'Label from git p4'
2729
2730         gitStream.write("data %d\n" % len(description))
2731         gitStream.write(description)
2732         gitStream.write("\n")
2733
2734     def inClientSpec(self, path):
2735         if not self.clientSpecDirs:
2736             return True
2737         inClientSpec = self.clientSpecDirs.map_in_client(path)
2738         if not inClientSpec and self.verbose:
2739             print('Ignoring file outside of client spec: {0}'.format(path))
2740         return inClientSpec
2741
2742     def hasBranchPrefix(self, path):
2743         if not self.branchPrefixes:
2744             return True
2745         hasPrefix = [p for p in self.branchPrefixes
2746                         if p4PathStartsWith(path, p)]
2747         if not hasPrefix and self.verbose:
2748             print('Ignoring file outside of prefix: {0}'.format(path))
2749         return hasPrefix
2750
2751     def commit(self, details, files, branch, parent = ""):
2752         epoch = details["time"]
2753         author = details["user"]
2754         jobs = self.extractJobsFromCommit(details)
2755
2756         if self.verbose:
2757             print('commit into {0}'.format(branch))
2758
2759         if self.clientSpecDirs:
2760             self.clientSpecDirs.update_client_spec_path_cache(files)
2761
2762         files = [f for f in files
2763             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2764
2765         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2766             print('Ignoring revision {0} as it would produce an empty commit.'
2767                 .format(details['change']))
2768             return
2769
2770         self.gitStream.write("commit %s\n" % branch)
2771         self.gitStream.write("mark :%s\n" % details["change"])
2772         self.committedChanges.add(int(details["change"]))
2773         committer = ""
2774         if author not in self.users:
2775             self.getUserMapFromPerforceServer()
2776         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2777
2778         self.gitStream.write("committer %s\n" % committer)
2779
2780         self.gitStream.write("data <<EOT\n")
2781         self.gitStream.write(details["desc"])
2782         if len(jobs) > 0:
2783             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2784         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2785                              (','.join(self.branchPrefixes), details["change"]))
2786         if len(details['options']) > 0:
2787             self.gitStream.write(": options = %s" % details['options'])
2788         self.gitStream.write("]\nEOT\n\n")
2789
2790         if len(parent) > 0:
2791             if self.verbose:
2792                 print "parent %s" % parent
2793             self.gitStream.write("from %s\n" % parent)
2794
2795         self.streamP4Files(files)
2796         self.gitStream.write("\n")
2797
2798         change = int(details["change"])
2799
2800         if self.labels.has_key(change):
2801             label = self.labels[change]
2802             labelDetails = label[0]
2803             labelRevisions = label[1]
2804             if self.verbose:
2805                 print "Change %s is labelled %s" % (change, labelDetails)
2806
2807             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2808                                                 for p in self.branchPrefixes])
2809
2810             if len(files) == len(labelRevisions):
2811
2812                 cleanedFiles = {}
2813                 for info in files:
2814                     if info["action"] in self.delete_actions:
2815                         continue
2816                     cleanedFiles[info["depotFile"]] = info["rev"]
2817
2818                 if cleanedFiles == labelRevisions:
2819                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2820
2821                 else:
2822                     if not self.silent:
2823                         print ("Tag %s does not match with change %s: files do not match."
2824                                % (labelDetails["label"], change))
2825
2826             else:
2827                 if not self.silent:
2828                     print ("Tag %s does not match with change %s: file count is different."
2829                            % (labelDetails["label"], change))
2830
2831     # Build a dictionary of changelists and labels, for "detect-labels" option.
2832     def getLabels(self):
2833         self.labels = {}
2834
2835         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2836         if len(l) > 0 and not self.silent:
2837             print "Finding files belonging to labels in %s" % `self.depotPaths`
2838
2839         for output in l:
2840             label = output["label"]
2841             revisions = {}
2842             newestChange = 0
2843             if self.verbose:
2844                 print "Querying files for label %s" % label
2845             for file in p4CmdList(["files"] +
2846                                       ["%s...@%s" % (p, label)
2847                                           for p in self.depotPaths]):
2848                 revisions[file["depotFile"]] = file["rev"]
2849                 change = int(file["change"])
2850                 if change > newestChange:
2851                     newestChange = change
2852
2853             self.labels[newestChange] = [output, revisions]
2854
2855         if self.verbose:
2856             print "Label changes: %s" % self.labels.keys()
2857
2858     # Import p4 labels as git tags. A direct mapping does not
2859     # exist, so assume that if all the files are at the same revision
2860     # then we can use that, or it's something more complicated we should
2861     # just ignore.
2862     def importP4Labels(self, stream, p4Labels):
2863         if verbose:
2864             print "import p4 labels: " + ' '.join(p4Labels)
2865
2866         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2867         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2868         if len(validLabelRegexp) == 0:
2869             validLabelRegexp = defaultLabelRegexp
2870         m = re.compile(validLabelRegexp)
2871
2872         for name in p4Labels:
2873             commitFound = False
2874
2875             if not m.match(name):
2876                 if verbose:
2877                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2878                 continue
2879
2880             if name in ignoredP4Labels:
2881                 continue
2882
2883             labelDetails = p4CmdList(['label', "-o", name])[0]
2884
2885             # get the most recent changelist for each file in this label
2886             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2887                                 for p in self.depotPaths])
2888
2889             if change.has_key('change'):
2890                 # find the corresponding git commit; take the oldest commit
2891                 changelist = int(change['change'])
2892                 if changelist in self.committedChanges:
2893                     gitCommit = ":%d" % changelist       # use a fast-import mark
2894                     commitFound = True
2895                 else:
2896                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2897                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2898                     if len(gitCommit) == 0:
2899                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2900                     else:
2901                         commitFound = True
2902                         gitCommit = gitCommit.strip()
2903
2904                 if commitFound:
2905                     # Convert from p4 time format
2906                     try:
2907                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2908                     except ValueError:
2909                         print "Could not convert label time %s" % labelDetails['Update']
2910                         tmwhen = 1
2911
2912                     when = int(time.mktime(tmwhen))
2913                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2914                     if verbose:
2915                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2916             else:
2917                 if verbose:
2918                     print "Label %s has no changelists - possibly deleted?" % name
2919
2920             if not commitFound:
2921                 # We can't import this label; don't try again as it will get very
2922                 # expensive repeatedly fetching all the files for labels that will
2923                 # never be imported. If the label is moved in the future, the
2924                 # ignore will need to be removed manually.
2925                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2926
2927     def guessProjectName(self):
2928         for p in self.depotPaths:
2929             if p.endswith("/"):
2930                 p = p[:-1]
2931             p = p[p.strip().rfind("/") + 1:]
2932             if not p.endswith("/"):
2933                p += "/"
2934             return p
2935
2936     def getBranchMapping(self):
2937         lostAndFoundBranches = set()
2938
2939         user = gitConfig("git-p4.branchUser")
2940         if len(user) > 0:
2941             command = "branches -u %s" % user
2942         else:
2943             command = "branches"
2944
2945         for info in p4CmdList(command):
2946             details = p4Cmd(["branch", "-o", info["branch"]])
2947             viewIdx = 0
2948             while details.has_key("View%s" % viewIdx):
2949                 paths = details["View%s" % viewIdx].split(" ")
2950                 viewIdx = viewIdx + 1
2951                 # require standard //depot/foo/... //depot/bar/... mapping
2952                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2953                     continue
2954                 source = paths[0]
2955                 destination = paths[1]
2956                 ## HACK
2957                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2958                     source = source[len(self.depotPaths[0]):-4]
2959                     destination = destination[len(self.depotPaths[0]):-4]
2960
2961                     if destination in self.knownBranches:
2962                         if not self.silent:
2963                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2964                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2965                         continue
2966
2967                     self.knownBranches[destination] = source
2968
2969                     lostAndFoundBranches.discard(destination)
2970
2971                     if source not in self.knownBranches:
2972                         lostAndFoundBranches.add(source)
2973
2974         # Perforce does not strictly require branches to be defined, so we also
2975         # check git config for a branch list.
2976         #
2977         # Example of branch definition in git config file:
2978         # [git-p4]
2979         #   branchList=main:branchA
2980         #   branchList=main:branchB
2981         #   branchList=branchA:branchC
2982         configBranches = gitConfigList("git-p4.branchList")
2983         for branch in configBranches:
2984             if branch:
2985                 (source, destination) = branch.split(":")
2986                 self.knownBranches[destination] = source
2987
2988                 lostAndFoundBranches.discard(destination)
2989
2990                 if source not in self.knownBranches:
2991                     lostAndFoundBranches.add(source)
2992
2993
2994         for branch in lostAndFoundBranches:
2995             self.knownBranches[branch] = branch
2996
2997     def getBranchMappingFromGitBranches(self):
2998         branches = p4BranchesInGit(self.importIntoRemotes)
2999         for branch in branches.keys():
3000             if branch == "master":
3001                 branch = "main"
3002             else:
3003                 branch = branch[len(self.projectName):]
3004             self.knownBranches[branch] = branch
3005
3006     def updateOptionDict(self, d):
3007         option_keys = {}
3008         if self.keepRepoPath:
3009             option_keys['keepRepoPath'] = 1
3010
3011         d["options"] = ' '.join(sorted(option_keys.keys()))
3012
3013     def readOptions(self, d):
3014         self.keepRepoPath = (d.has_key('options')
3015                              and ('keepRepoPath' in d['options']))
3016
3017     def gitRefForBranch(self, branch):
3018         if branch == "main":
3019             return self.refPrefix + "master"
3020
3021         if len(branch) <= 0:
3022             return branch
3023
3024         return self.refPrefix + self.projectName + branch
3025
3026     def gitCommitByP4Change(self, ref, change):
3027         if self.verbose:
3028             print "looking in ref " + ref + " for change %s using bisect..." % change
3029
3030         earliestCommit = ""
3031         latestCommit = parseRevision(ref)
3032
3033         while True:
3034             if self.verbose:
3035                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3036             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3037             if len(next) == 0:
3038                 if self.verbose:
3039                     print "argh"
3040                 return ""
3041             log = extractLogMessageFromGitCommit(next)
3042             settings = extractSettingsGitLog(log)
3043             currentChange = int(settings['change'])
3044             if self.verbose:
3045                 print "current change %s" % currentChange
3046
3047             if currentChange == change:
3048                 if self.verbose:
3049                     print "found %s" % next
3050                 return next
3051
3052             if currentChange < change:
3053                 earliestCommit = "^%s" % next
3054             else:
3055                 latestCommit = "%s" % next
3056
3057         return ""
3058
3059     def importNewBranch(self, branch, maxChange):
3060         # make fast-import flush all changes to disk and update the refs using the checkpoint
3061         # command so that we can try to find the branch parent in the git history
3062         self.gitStream.write("checkpoint\n\n");
3063         self.gitStream.flush();
3064         branchPrefix = self.depotPaths[0] + branch + "/"
3065         range = "@1,%s" % maxChange
3066         #print "prefix" + branchPrefix
3067         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3068         if len(changes) <= 0:
3069             return False
3070         firstChange = changes[0]
3071         #print "first change in branch: %s" % firstChange
3072         sourceBranch = self.knownBranches[branch]
3073         sourceDepotPath = self.depotPaths[0] + sourceBranch
3074         sourceRef = self.gitRefForBranch(sourceBranch)
3075         #print "source " + sourceBranch
3076
3077         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3078         #print "branch parent: %s" % branchParentChange
3079         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3080         if len(gitParent) > 0:
3081             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3082             #print "parent git commit: %s" % gitParent
3083
3084         self.importChanges(changes)
3085         return True
3086
3087     def searchParent(self, parent, branch, target):
3088         parentFound = False
3089         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3090                                      "--no-merges", parent]):
3091             blob = blob.strip()
3092             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3093                 parentFound = True
3094                 if self.verbose:
3095                     print "Found parent of %s in commit %s" % (branch, blob)
3096                 break
3097         if parentFound:
3098             return blob
3099         else:
3100             return None
3101
3102     def importChanges(self, changes):
3103         cnt = 1
3104         for change in changes:
3105             description = p4_describe(change)
3106             self.updateOptionDict(description)
3107
3108             if not self.silent:
3109                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3110                 sys.stdout.flush()
3111             cnt = cnt + 1
3112
3113             try:
3114                 if self.detectBranches:
3115                     branches = self.splitFilesIntoBranches(description)
3116                     for branch in branches.keys():
3117                         ## HACK  --hwn
3118                         branchPrefix = self.depotPaths[0] + branch + "/"
3119                         self.branchPrefixes = [ branchPrefix ]
3120
3121                         parent = ""
3122
3123                         filesForCommit = branches[branch]
3124
3125                         if self.verbose:
3126                             print "branch is %s" % branch
3127
3128                         self.updatedBranches.add(branch)
3129
3130                         if branch not in self.createdBranches:
3131                             self.createdBranches.add(branch)
3132                             parent = self.knownBranches[branch]
3133                             if parent == branch:
3134                                 parent = ""
3135                             else:
3136                                 fullBranch = self.projectName + branch
3137                                 if fullBranch not in self.p4BranchesInGit:
3138                                     if not self.silent:
3139                                         print("\n    Importing new branch %s" % fullBranch);
3140                                     if self.importNewBranch(branch, change - 1):
3141                                         parent = ""
3142                                         self.p4BranchesInGit.append(fullBranch)
3143                                     if not self.silent:
3144                                         print("\n    Resuming with change %s" % change);
3145
3146                                 if self.verbose:
3147                                     print "parent determined through known branches: %s" % parent
3148
3149                         branch = self.gitRefForBranch(branch)
3150                         parent = self.gitRefForBranch(parent)
3151
3152                         if self.verbose:
3153                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3154
3155                         if len(parent) == 0 and branch in self.initialParents:
3156                             parent = self.initialParents[branch]
3157                             del self.initialParents[branch]
3158
3159                         blob = None
3160                         if len(parent) > 0:
3161                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3162                             if self.verbose:
3163                                 print "Creating temporary branch: " + tempBranch
3164                             self.commit(description, filesForCommit, tempBranch)
3165                             self.tempBranches.append(tempBranch)
3166                             self.checkpoint()
3167                             blob = self.searchParent(parent, branch, tempBranch)
3168                         if blob:
3169                             self.commit(description, filesForCommit, branch, blob)
3170                         else:
3171                             if self.verbose:
3172                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3173                             self.commit(description, filesForCommit, branch, parent)
3174                 else:
3175                     files = self.extractFilesFromCommit(description)
3176                     self.commit(description, files, self.branch,
3177                                 self.initialParent)
3178                     # only needed once, to connect to the previous commit
3179                     self.initialParent = ""
3180             except IOError:
3181                 print self.gitError.read()
3182                 sys.exit(1)
3183
3184     def importHeadRevision(self, revision):
3185         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3186
3187         details = {}
3188         details["user"] = "git perforce import user"
3189         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3190                            % (' '.join(self.depotPaths), revision))
3191         details["change"] = revision
3192         newestRevision = 0
3193
3194         fileCnt = 0
3195         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3196
3197         for info in p4CmdList(["files"] + fileArgs):
3198
3199             if 'code' in info and info['code'] == 'error':
3200                 sys.stderr.write("p4 returned an error: %s\n"
3201                                  % info['data'])
3202                 if info['data'].find("must refer to client") >= 0:
3203                     sys.stderr.write("This particular p4 error is misleading.\n")
3204                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3205                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3206                 sys.exit(1)
3207             if 'p4ExitCode' in info:
3208                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3209                 sys.exit(1)
3210
3211
3212             change = int(info["change"])
3213             if change > newestRevision:
3214                 newestRevision = change
3215
3216             if info["action"] in self.delete_actions:
3217                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3218                 #fileCnt = fileCnt + 1
3219                 continue
3220
3221             for prop in ["depotFile", "rev", "action", "type" ]:
3222                 details["%s%s" % (prop, fileCnt)] = info[prop]
3223
3224             fileCnt = fileCnt + 1
3225
3226         details["change"] = newestRevision
3227
3228         # Use time from top-most change so that all git p4 clones of
3229         # the same p4 repo have the same commit SHA1s.
3230         res = p4_describe(newestRevision)
3231         details["time"] = res["time"]
3232
3233         self.updateOptionDict(details)
3234         try:
3235             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3236         except IOError:
3237             print "IO error with git fast-import. Is your git version recent enough?"
3238             print self.gitError.read()
3239
3240
3241     def run(self, args):
3242         self.depotPaths = []
3243         self.changeRange = ""
3244         self.previousDepotPaths = []
3245         self.hasOrigin = False
3246
3247         # map from branch depot path to parent branch
3248         self.knownBranches = {}
3249         self.initialParents = {}
3250
3251         if self.importIntoRemotes:
3252             self.refPrefix = "refs/remotes/p4/"
3253         else:
3254             self.refPrefix = "refs/heads/p4/"
3255
3256         if self.syncWithOrigin:
3257             self.hasOrigin = originP4BranchesExist()
3258             if self.hasOrigin:
3259                 if not self.silent:
3260                     print 'Syncing with origin first, using "git fetch origin"'
3261                 system("git fetch origin")
3262
3263         branch_arg_given = bool(self.branch)
3264         if len(self.branch) == 0:
3265             self.branch = self.refPrefix + "master"
3266             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3267                 system("git update-ref %s refs/heads/p4" % self.branch)
3268                 system("git branch -D p4")
3269
3270         # accept either the command-line option, or the configuration variable
3271         if self.useClientSpec:
3272             # will use this after clone to set the variable
3273             self.useClientSpec_from_options = True
3274         else:
3275             if gitConfigBool("git-p4.useclientspec"):
3276                 self.useClientSpec = True
3277         if self.useClientSpec:
3278             self.clientSpecDirs = getClientSpec()
3279
3280         # TODO: should always look at previous commits,
3281         # merge with previous imports, if possible.
3282         if args == []:
3283             if self.hasOrigin:
3284                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3285
3286             # branches holds mapping from branch name to sha1
3287             branches = p4BranchesInGit(self.importIntoRemotes)
3288
3289             # restrict to just this one, disabling detect-branches
3290             if branch_arg_given:
3291                 short = self.branch.split("/")[-1]
3292                 if short in branches:
3293                     self.p4BranchesInGit = [ short ]
3294             else:
3295                 self.p4BranchesInGit = branches.keys()
3296
3297             if len(self.p4BranchesInGit) > 1:
3298                 if not self.silent:
3299                     print "Importing from/into multiple branches"
3300                 self.detectBranches = True
3301                 for branch in branches.keys():
3302                     self.initialParents[self.refPrefix + branch] = \
3303                         branches[branch]
3304
3305             if self.verbose:
3306                 print "branches: %s" % self.p4BranchesInGit
3307
3308             p4Change = 0
3309             for branch in self.p4BranchesInGit:
3310                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3311
3312                 settings = extractSettingsGitLog(logMsg)
3313
3314                 self.readOptions(settings)
3315                 if (settings.has_key('depot-paths')
3316                     and settings.has_key ('change')):
3317                     change = int(settings['change']) + 1
3318                     p4Change = max(p4Change, change)
3319
3320                     depotPaths = sorted(settings['depot-paths'])
3321                     if self.previousDepotPaths == []:
3322                         self.previousDepotPaths = depotPaths
3323                     else:
3324                         paths = []
3325                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3326                             prev_list = prev.split("/")
3327                             cur_list = cur.split("/")
3328                             for i in range(0, min(len(cur_list), len(prev_list))):
3329                                 if cur_list[i] <> prev_list[i]:
3330                                     i = i - 1
3331                                     break
3332
3333                             paths.append ("/".join(cur_list[:i + 1]))
3334
3335                         self.previousDepotPaths = paths
3336
3337             if p4Change > 0:
3338                 self.depotPaths = sorted(self.previousDepotPaths)
3339                 self.changeRange = "@%s,#head" % p4Change
3340                 if not self.silent and not self.detectBranches:
3341                     print "Performing incremental import into %s git branch" % self.branch
3342
3343         # accept multiple ref name abbreviations:
3344         #    refs/foo/bar/branch -> use it exactly
3345         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3346         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3347         if not self.branch.startswith("refs/"):
3348             if self.importIntoRemotes:
3349                 prepend = "refs/remotes/"
3350             else:
3351                 prepend = "refs/heads/"
3352             if not self.branch.startswith("p4/"):
3353                 prepend += "p4/"
3354             self.branch = prepend + self.branch
3355
3356         if len(args) == 0 and self.depotPaths:
3357             if not self.silent:
3358                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3359         else:
3360             if self.depotPaths and self.depotPaths != args:
3361                 print ("previous import used depot path %s and now %s was specified. "
3362                        "This doesn't work!" % (' '.join (self.depotPaths),
3363                                                ' '.join (args)))
3364                 sys.exit(1)
3365
3366             self.depotPaths = sorted(args)
3367
3368         revision = ""
3369         self.users = {}
3370
3371         # Make sure no revision specifiers are used when --changesfile
3372         # is specified.
3373         bad_changesfile = False
3374         if len(self.changesFile) > 0:
3375             for p in self.depotPaths:
3376                 if p.find("@") >= 0 or p.find("#") >= 0:
3377                     bad_changesfile = True
3378                     break
3379         if bad_changesfile:
3380             die("Option --changesfile is incompatible with revision specifiers")
3381
3382         newPaths = []
3383         for p in self.depotPaths:
3384             if p.find("@") != -1:
3385                 atIdx = p.index("@")
3386                 self.changeRange = p[atIdx:]
3387                 if self.changeRange == "@all":
3388                     self.changeRange = ""
3389                 elif ',' not in self.changeRange:
3390                     revision = self.changeRange
3391                     self.changeRange = ""
3392                 p = p[:atIdx]
3393             elif p.find("#") != -1:
3394                 hashIdx = p.index("#")
3395                 revision = p[hashIdx:]
3396                 p = p[:hashIdx]
3397             elif self.previousDepotPaths == []:
3398                 # pay attention to changesfile, if given, else import
3399                 # the entire p4 tree at the head revision
3400                 if len(self.changesFile) == 0:
3401                     revision = "#head"
3402
3403             p = re.sub ("\.\.\.$", "", p)
3404             if not p.endswith("/"):
3405                 p += "/"
3406
3407             newPaths.append(p)
3408
3409         self.depotPaths = newPaths
3410
3411         # --detect-branches may change this for each branch
3412         self.branchPrefixes = self.depotPaths
3413
3414         self.loadUserMapFromCache()
3415         self.labels = {}
3416         if self.detectLabels:
3417             self.getLabels();
3418
3419         if self.detectBranches:
3420             ## FIXME - what's a P4 projectName ?
3421             self.projectName = self.guessProjectName()
3422
3423             if self.hasOrigin:
3424                 self.getBranchMappingFromGitBranches()
3425             else:
3426                 self.getBranchMapping()
3427             if self.verbose:
3428                 print "p4-git branches: %s" % self.p4BranchesInGit
3429                 print "initial parents: %s" % self.initialParents
3430             for b in self.p4BranchesInGit:
3431                 if b != "master":
3432
3433                     ## FIXME
3434                     b = b[len(self.projectName):]
3435                 self.createdBranches.add(b)
3436
3437         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3438
3439         self.importProcess = subprocess.Popen(["git", "fast-import"],
3440                                               stdin=subprocess.PIPE,
3441                                               stdout=subprocess.PIPE,
3442                                               stderr=subprocess.PIPE);
3443         self.gitOutput = self.importProcess.stdout
3444         self.gitStream = self.importProcess.stdin
3445         self.gitError = self.importProcess.stderr
3446
3447         if revision:
3448             self.importHeadRevision(revision)
3449         else:
3450             changes = []
3451
3452             if len(self.changesFile) > 0:
3453                 output = open(self.changesFile).readlines()
3454                 changeSet = set()
3455                 for line in output:
3456                     changeSet.add(int(line))
3457
3458                 for change in changeSet:
3459                     changes.append(change)
3460
3461                 changes.sort()
3462             else:
3463                 # catch "git p4 sync" with no new branches, in a repo that
3464                 # does not have any existing p4 branches
3465                 if len(args) == 0:
3466                     if not self.p4BranchesInGit:
3467                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3468
3469                     # The default branch is master, unless --branch is used to
3470                     # specify something else.  Make sure it exists, or complain
3471                     # nicely about how to use --branch.
3472                     if not self.detectBranches:
3473                         if not branch_exists(self.branch):
3474                             if branch_arg_given:
3475                                 die("Error: branch %s does not exist." % self.branch)
3476                             else:
3477                                 die("Error: no branch %s; perhaps specify one with --branch." %
3478                                     self.branch)
3479
3480                 if self.verbose:
3481                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3482                                                               self.changeRange)
3483                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3484
3485                 if len(self.maxChanges) > 0:
3486                     changes = changes[:min(int(self.maxChanges), len(changes))]
3487
3488             if len(changes) == 0:
3489                 if not self.silent:
3490                     print "No changes to import!"
3491             else:
3492                 if not self.silent and not self.detectBranches:
3493                     print "Import destination: %s" % self.branch
3494
3495                 self.updatedBranches = set()
3496
3497                 if not self.detectBranches:
3498                     if args:
3499                         # start a new branch
3500                         self.initialParent = ""
3501                     else:
3502                         # build on a previous revision
3503                         self.initialParent = parseRevision(self.branch)
3504
3505                 self.importChanges(changes)
3506
3507                 if not self.silent:
3508                     print ""
3509                     if len(self.updatedBranches) > 0:
3510                         sys.stdout.write("Updated branches: ")
3511                         for b in self.updatedBranches:
3512                             sys.stdout.write("%s " % b)
3513                         sys.stdout.write("\n")
3514
3515         if gitConfigBool("git-p4.importLabels"):
3516             self.importLabels = True
3517
3518         if self.importLabels:
3519             p4Labels = getP4Labels(self.depotPaths)
3520             gitTags = getGitTags()
3521
3522             missingP4Labels = p4Labels - gitTags
3523             self.importP4Labels(self.gitStream, missingP4Labels)
3524
3525         self.gitStream.close()
3526         if self.importProcess.wait() != 0:
3527             die("fast-import failed: %s" % self.gitError.read())
3528         self.gitOutput.close()
3529         self.gitError.close()
3530
3531         # Cleanup temporary branches created during import
3532         if self.tempBranches != []:
3533             for branch in self.tempBranches:
3534                 read_pipe("git update-ref -d %s" % branch)
3535             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3536
3537         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3538         # a convenient shortcut refname "p4".
3539         if self.importIntoRemotes:
3540             head_ref = self.refPrefix + "HEAD"
3541             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3542                 system(["git", "symbolic-ref", head_ref, self.branch])
3543
3544         return True
3545
3546 class P4Rebase(Command):
3547     def __init__(self):
3548         Command.__init__(self)
3549         self.options = [
3550                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3551         ]
3552         self.importLabels = False
3553         self.description = ("Fetches the latest revision from perforce and "
3554                             + "rebases the current work (branch) against it")
3555
3556     def run(self, args):
3557         sync = P4Sync()
3558         sync.importLabels = self.importLabels
3559         sync.run([])
3560
3561         return self.rebase()
3562
3563     def rebase(self):
3564         if os.system("git update-index --refresh") != 0:
3565             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3566         if len(read_pipe("git diff-index HEAD --")) > 0:
3567             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3568
3569         [upstream, settings] = findUpstreamBranchPoint()
3570         if len(upstream) == 0:
3571             die("Cannot find upstream branchpoint for rebase")
3572
3573         # the branchpoint may be p4/foo~3, so strip off the parent
3574         upstream = re.sub("~[0-9]+$", "", upstream)
3575
3576         print "Rebasing the current branch onto %s" % upstream
3577         oldHead = read_pipe("git rev-parse HEAD").strip()
3578         system("git rebase %s" % upstream)
3579         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3580         return True
3581
3582 class P4Clone(P4Sync):
3583     def __init__(self):
3584         P4Sync.__init__(self)
3585         self.description = "Creates a new git repository and imports from Perforce into it"
3586         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3587         self.options += [
3588             optparse.make_option("--destination", dest="cloneDestination",
3589                                  action='store', default=None,
3590                                  help="where to leave result of the clone"),
3591             optparse.make_option("--bare", dest="cloneBare",
3592                                  action="store_true", default=False),
3593         ]
3594         self.cloneDestination = None
3595         self.needsGit = False
3596         self.cloneBare = False
3597
3598     def defaultDestination(self, args):
3599         ## TODO: use common prefix of args?
3600         depotPath = args[0]
3601         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3602         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3603         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3604         depotDir = re.sub(r"/$", "", depotDir)
3605         return os.path.split(depotDir)[1]
3606
3607     def run(self, args):
3608         if len(args) < 1:
3609             return False
3610
3611         if self.keepRepoPath and not self.cloneDestination:
3612             sys.stderr.write("Must specify destination for --keep-path\n")
3613             sys.exit(1)
3614
3615         depotPaths = args
3616
3617         if not self.cloneDestination and len(depotPaths) > 1:
3618             self.cloneDestination = depotPaths[-1]
3619             depotPaths = depotPaths[:-1]
3620
3621         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3622         for p in depotPaths:
3623             if not p.startswith("//"):
3624                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3625                 return False
3626
3627         if not self.cloneDestination:
3628             self.cloneDestination = self.defaultDestination(args)
3629
3630         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3631
3632         if not os.path.exists(self.cloneDestination):
3633             os.makedirs(self.cloneDestination)
3634         chdir(self.cloneDestination)
3635
3636         init_cmd = [ "git", "init" ]
3637         if self.cloneBare:
3638             init_cmd.append("--bare")
3639         retcode = subprocess.call(init_cmd)
3640         if retcode:
3641             raise CalledProcessError(retcode, init_cmd)
3642
3643         if not P4Sync.run(self, depotPaths):
3644             return False
3645
3646         # create a master branch and check out a work tree
3647         if gitBranchExists(self.branch):
3648             system([ "git", "branch", "master", self.branch ])
3649             if not self.cloneBare:
3650                 system([ "git", "checkout", "-f" ])
3651         else:
3652             print 'Not checking out any branch, use ' \
3653                   '"git checkout -q -b master <branch>"'
3654
3655         # auto-set this variable if invoked with --use-client-spec
3656         if self.useClientSpec_from_options:
3657             system("git config --bool git-p4.useclientspec true")
3658
3659         return True
3660
3661 class P4Branches(Command):
3662     def __init__(self):
3663         Command.__init__(self)
3664         self.options = [ ]
3665         self.description = ("Shows the git branches that hold imports and their "
3666                             + "corresponding perforce depot paths")
3667         self.verbose = False
3668
3669     def run(self, args):
3670         if originP4BranchesExist():
3671             createOrUpdateBranchesFromOrigin()
3672
3673         cmdline = "git rev-parse --symbolic "
3674         cmdline += " --remotes"
3675
3676         for line in read_pipe_lines(cmdline):
3677             line = line.strip()
3678
3679             if not line.startswith('p4/') or line == "p4/HEAD":
3680                 continue
3681             branch = line
3682
3683             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3684             settings = extractSettingsGitLog(log)
3685
3686             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3687         return True
3688
3689 class HelpFormatter(optparse.IndentedHelpFormatter):
3690     def __init__(self):
3691         optparse.IndentedHelpFormatter.__init__(self)
3692
3693     def format_description(self, description):
3694         if description:
3695             return description + "\n"
3696         else:
3697             return ""
3698
3699 def printUsage(commands):
3700     print "usage: %s <command> [options]" % sys.argv[0]
3701     print ""
3702     print "valid commands: %s" % ", ".join(commands)
3703     print ""
3704     print "Try %s <command> --help for command specific help." % sys.argv[0]
3705     print ""
3706
3707 commands = {
3708     "debug" : P4Debug,
3709     "submit" : P4Submit,
3710     "commit" : P4Submit,
3711     "sync" : P4Sync,
3712     "rebase" : P4Rebase,
3713     "clone" : P4Clone,
3714     "rollback" : P4RollBack,
3715     "branches" : P4Branches
3716 }
3717
3718
3719 def main():
3720     if len(sys.argv[1:]) == 0:
3721         printUsage(commands.keys())
3722         sys.exit(2)
3723
3724     cmdName = sys.argv[1]
3725     try:
3726         klass = commands[cmdName]
3727         cmd = klass()
3728     except KeyError:
3729         print "unknown command %s" % cmdName
3730         print ""
3731         printUsage(commands.keys())
3732         sys.exit(2)
3733
3734     options = cmd.options
3735     cmd.gitdir = os.environ.get("GIT_DIR", None)
3736
3737     args = sys.argv[2:]
3738
3739     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3740     if cmd.needsGit:
3741         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3742
3743     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3744                                    options,
3745                                    description = cmd.description,
3746                                    formatter = HelpFormatter())
3747
3748     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3749     global verbose
3750     verbose = cmd.verbose
3751     if cmd.needsGit:
3752         if cmd.gitdir == None:
3753             cmd.gitdir = os.path.abspath(".git")
3754             if not isValidGitDir(cmd.gitdir):
3755                 # "rev-parse --git-dir" without arguments will try $PWD/.git
3756                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3757                 if os.path.exists(cmd.gitdir):
3758                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3759                     if len(cdup) > 0:
3760                         chdir(cdup);
3761
3762         if not isValidGitDir(cmd.gitdir):
3763             if isValidGitDir(cmd.gitdir + "/.git"):
3764                 cmd.gitdir += "/.git"
3765             else:
3766                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3767
3768         # so git commands invoked from the P4 workspace will succeed
3769         os.environ["GIT_DIR"] = cmd.gitdir
3770
3771     if not cmd.run(args):
3772         parser.print_help()
3773         sys.exit(2)
3774
3775
3776 if __name__ == '__main__':
3777     main()