Merge branch 'dt/t7063-fix-flaky-test' into maint
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27     from subprocess import CalledProcessError
28 except ImportError:
29     # from python2.7:subprocess.py
30     # Exception classes used by this module.
31     class CalledProcessError(Exception):
32         """This exception is raised when a process run by check_call() returns
33         a non-zero exit status.  The exit status will be stored in the
34         returncode attribute."""
35         def __init__(self, returncode, cmd):
36             self.returncode = returncode
37             self.cmd = cmd
38         def __str__(self):
39             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 # Grab changes in blocks of this many revisions, unless otherwise requested
47 defaultBlockSize = 512
48
49 def p4_build_cmd(cmd):
50     """Build a suitable p4 command line.
51
52     This consolidates building and returning a p4 command line into one
53     location. It means that hooking into the environment, or other configuration
54     can be done more easily.
55     """
56     real_cmd = ["p4"]
57
58     user = gitConfig("git-p4.user")
59     if len(user) > 0:
60         real_cmd += ["-u",user]
61
62     password = gitConfig("git-p4.password")
63     if len(password) > 0:
64         real_cmd += ["-P", password]
65
66     port = gitConfig("git-p4.port")
67     if len(port) > 0:
68         real_cmd += ["-p", port]
69
70     host = gitConfig("git-p4.host")
71     if len(host) > 0:
72         real_cmd += ["-H", host]
73
74     client = gitConfig("git-p4.client")
75     if len(client) > 0:
76         real_cmd += ["-c", client]
77
78
79     if isinstance(cmd,basestring):
80         real_cmd = ' '.join(real_cmd) + ' ' + cmd
81     else:
82         real_cmd += cmd
83     return real_cmd
84
85 def chdir(path, is_client_path=False):
86     """Do chdir to the given path, and set the PWD environment
87        variable for use by P4.  It does not look at getcwd() output.
88        Since we're not using the shell, it is necessary to set the
89        PWD environment variable explicitly.
90
91        Normally, expand the path to force it to be absolute.  This
92        addresses the use of relative path names inside P4 settings,
93        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
94        as given; it looks for .p4config using PWD.
95
96        If is_client_path, the path was handed to us directly by p4,
97        and may be a symbolic link.  Do not call os.getcwd() in this
98        case, because it will cause p4 to think that PWD is not inside
99        the client path.
100        """
101
102     os.chdir(path)
103     if not is_client_path:
104         path = os.getcwd()
105     os.environ['PWD'] = path
106
107 def die(msg):
108     if verbose:
109         raise Exception(msg)
110     else:
111         sys.stderr.write(msg + "\n")
112         sys.exit(1)
113
114 def write_pipe(c, stdin):
115     if verbose:
116         sys.stderr.write('Writing pipe: %s\n' % str(c))
117
118     expand = isinstance(c,basestring)
119     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
120     pipe = p.stdin
121     val = pipe.write(stdin)
122     pipe.close()
123     if p.wait():
124         die('Command failed: %s' % str(c))
125
126     return val
127
128 def p4_write_pipe(c, stdin):
129     real_cmd = p4_build_cmd(c)
130     return write_pipe(real_cmd, stdin)
131
132 def read_pipe(c, ignore_error=False):
133     if verbose:
134         sys.stderr.write('Reading pipe: %s\n' % str(c))
135
136     expand = isinstance(c,basestring)
137     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
138     (out, err) = p.communicate()
139     if p.returncode != 0 and not ignore_error:
140         die('Command failed: %s\nError: %s' % (str(c), err))
141     return out
142
143 def p4_read_pipe(c, ignore_error=False):
144     real_cmd = p4_build_cmd(c)
145     return read_pipe(real_cmd, ignore_error)
146
147 def read_pipe_lines(c):
148     if verbose:
149         sys.stderr.write('Reading pipe: %s\n' % str(c))
150
151     expand = isinstance(c, basestring)
152     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
153     pipe = p.stdout
154     val = pipe.readlines()
155     if pipe.close() or p.wait():
156         die('Command failed: %s' % str(c))
157
158     return val
159
160 def p4_read_pipe_lines(c):
161     """Specifically invoke p4 on the command supplied. """
162     real_cmd = p4_build_cmd(c)
163     return read_pipe_lines(real_cmd)
164
165 def p4_has_command(cmd):
166     """Ask p4 for help on this command.  If it returns an error, the
167        command does not exist in this version of p4."""
168     real_cmd = p4_build_cmd(["help", cmd])
169     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
170                                    stderr=subprocess.PIPE)
171     p.communicate()
172     return p.returncode == 0
173
174 def p4_has_move_command():
175     """See if the move command exists, that it supports -k, and that
176        it has not been administratively disabled.  The arguments
177        must be correct, but the filenames do not have to exist.  Use
178        ones with wildcards so even if they exist, it will fail."""
179
180     if not p4_has_command("move"):
181         return False
182     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
183     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
184     (out, err) = p.communicate()
185     # return code will be 1 in either case
186     if err.find("Invalid option") >= 0:
187         return False
188     if err.find("disabled") >= 0:
189         return False
190     # assume it failed because @... was invalid changelist
191     return True
192
193 def system(cmd):
194     expand = isinstance(cmd,basestring)
195     if verbose:
196         sys.stderr.write("executing %s\n" % str(cmd))
197     retcode = subprocess.call(cmd, shell=expand)
198     if retcode:
199         raise CalledProcessError(retcode, cmd)
200
201 def p4_system(cmd):
202     """Specifically invoke p4 as the system command. """
203     real_cmd = p4_build_cmd(cmd)
204     expand = isinstance(real_cmd, basestring)
205     retcode = subprocess.call(real_cmd, shell=expand)
206     if retcode:
207         raise CalledProcessError(retcode, real_cmd)
208
209 _p4_version_string = None
210 def p4_version_string():
211     """Read the version string, showing just the last line, which
212        hopefully is the interesting version bit.
213
214        $ p4 -V
215        Perforce - The Fast Software Configuration Management System.
216        Copyright 1995-2011 Perforce Software.  All rights reserved.
217        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
218     """
219     global _p4_version_string
220     if not _p4_version_string:
221         a = p4_read_pipe_lines(["-V"])
222         _p4_version_string = a[-1].rstrip()
223     return _p4_version_string
224
225 def p4_integrate(src, dest):
226     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
227
228 def p4_sync(f, *options):
229     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
230
231 def p4_add(f):
232     # forcibly add file names with wildcards
233     if wildcard_present(f):
234         p4_system(["add", "-f", f])
235     else:
236         p4_system(["add", f])
237
238 def p4_delete(f):
239     p4_system(["delete", wildcard_encode(f)])
240
241 def p4_edit(f):
242     p4_system(["edit", wildcard_encode(f)])
243
244 def p4_revert(f):
245     p4_system(["revert", wildcard_encode(f)])
246
247 def p4_reopen(type, f):
248     p4_system(["reopen", "-t", type, wildcard_encode(f)])
249
250 def p4_move(src, dest):
251     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
252
253 def p4_last_change():
254     results = p4CmdList(["changes", "-m", "1"])
255     return int(results[0]['change'])
256
257 def p4_describe(change):
258     """Make sure it returns a valid result by checking for
259        the presence of field "time".  Return a dict of the
260        results."""
261
262     ds = p4CmdList(["describe", "-s", str(change)])
263     if len(ds) != 1:
264         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
265
266     d = ds[0]
267
268     if "p4ExitCode" in d:
269         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
270                                                       str(d)))
271     if "code" in d:
272         if d["code"] == "error":
273             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
274
275     if "time" not in d:
276         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
277
278     return d
279
280 #
281 # Canonicalize the p4 type and return a tuple of the
282 # base type, plus any modifiers.  See "p4 help filetypes"
283 # for a list and explanation.
284 #
285 def split_p4_type(p4type):
286
287     p4_filetypes_historical = {
288         "ctempobj": "binary+Sw",
289         "ctext": "text+C",
290         "cxtext": "text+Cx",
291         "ktext": "text+k",
292         "kxtext": "text+kx",
293         "ltext": "text+F",
294         "tempobj": "binary+FSw",
295         "ubinary": "binary+F",
296         "uresource": "resource+F",
297         "uxbinary": "binary+Fx",
298         "xbinary": "binary+x",
299         "xltext": "text+Fx",
300         "xtempobj": "binary+Swx",
301         "xtext": "text+x",
302         "xunicode": "unicode+x",
303         "xutf16": "utf16+x",
304     }
305     if p4type in p4_filetypes_historical:
306         p4type = p4_filetypes_historical[p4type]
307     mods = ""
308     s = p4type.split("+")
309     base = s[0]
310     mods = ""
311     if len(s) > 1:
312         mods = s[1]
313     return (base, mods)
314
315 #
316 # return the raw p4 type of a file (text, text+ko, etc)
317 #
318 def p4_type(f):
319     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
320     return results[0]['headType']
321
322 #
323 # Given a type base and modifier, return a regexp matching
324 # the keywords that can be expanded in the file
325 #
326 def p4_keywords_regexp_for_type(base, type_mods):
327     if base in ("text", "unicode", "binary"):
328         kwords = None
329         if "ko" in type_mods:
330             kwords = 'Id|Header'
331         elif "k" in type_mods:
332             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
333         else:
334             return None
335         pattern = r"""
336             \$              # Starts with a dollar, followed by...
337             (%s)            # one of the keywords, followed by...
338             (:[^$\n]+)?     # possibly an old expansion, followed by...
339             \$              # another dollar
340             """ % kwords
341         return pattern
342     else:
343         return None
344
345 #
346 # Given a file, return a regexp matching the possible
347 # RCS keywords that will be expanded, or None for files
348 # with kw expansion turned off.
349 #
350 def p4_keywords_regexp_for_file(file):
351     if not os.path.exists(file):
352         return None
353     else:
354         (type_base, type_mods) = split_p4_type(p4_type(file))
355         return p4_keywords_regexp_for_type(type_base, type_mods)
356
357 def setP4ExecBit(file, mode):
358     # Reopens an already open file and changes the execute bit to match
359     # the execute bit setting in the passed in mode.
360
361     p4Type = "+x"
362
363     if not isModeExec(mode):
364         p4Type = getP4OpenedType(file)
365         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
366         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
367         if p4Type[-1] == "+":
368             p4Type = p4Type[0:-1]
369
370     p4_reopen(p4Type, file)
371
372 def getP4OpenedType(file):
373     # Returns the perforce file type for the given file.
374
375     result = p4_read_pipe(["opened", wildcard_encode(file)])
376     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
377     if match:
378         return match.group(1)
379     else:
380         die("Could not determine file type for %s (result: '%s')" % (file, result))
381
382 # Return the set of all p4 labels
383 def getP4Labels(depotPaths):
384     labels = set()
385     if isinstance(depotPaths,basestring):
386         depotPaths = [depotPaths]
387
388     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
389         label = l['label']
390         labels.add(label)
391
392     return labels
393
394 # Return the set of all git tags
395 def getGitTags():
396     gitTags = set()
397     for line in read_pipe_lines(["git", "tag"]):
398         tag = line.strip()
399         gitTags.add(tag)
400     return gitTags
401
402 def diffTreePattern():
403     # This is a simple generator for the diff tree regex pattern. This could be
404     # a class variable if this and parseDiffTreeEntry were a part of a class.
405     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
406     while True:
407         yield pattern
408
409 def parseDiffTreeEntry(entry):
410     """Parses a single diff tree entry into its component elements.
411
412     See git-diff-tree(1) manpage for details about the format of the diff
413     output. This method returns a dictionary with the following elements:
414
415     src_mode - The mode of the source file
416     dst_mode - The mode of the destination file
417     src_sha1 - The sha1 for the source file
418     dst_sha1 - The sha1 fr the destination file
419     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
420     status_score - The score for the status (applicable for 'C' and 'R'
421                    statuses). This is None if there is no score.
422     src - The path for the source file.
423     dst - The path for the destination file. This is only present for
424           copy or renames. If it is not present, this is None.
425
426     If the pattern is not matched, None is returned."""
427
428     match = diffTreePattern().next().match(entry)
429     if match:
430         return {
431             'src_mode': match.group(1),
432             'dst_mode': match.group(2),
433             'src_sha1': match.group(3),
434             'dst_sha1': match.group(4),
435             'status': match.group(5),
436             'status_score': match.group(6),
437             'src': match.group(7),
438             'dst': match.group(10)
439         }
440     return None
441
442 def isModeExec(mode):
443     # Returns True if the given git mode represents an executable file,
444     # otherwise False.
445     return mode[-3:] == "755"
446
447 def isModeExecChanged(src_mode, dst_mode):
448     return isModeExec(src_mode) != isModeExec(dst_mode)
449
450 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
451
452     if isinstance(cmd,basestring):
453         cmd = "-G " + cmd
454         expand = True
455     else:
456         cmd = ["-G"] + cmd
457         expand = False
458
459     cmd = p4_build_cmd(cmd)
460     if verbose:
461         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
462
463     # Use a temporary file to avoid deadlocks without
464     # subprocess.communicate(), which would put another copy
465     # of stdout into memory.
466     stdin_file = None
467     if stdin is not None:
468         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
469         if isinstance(stdin,basestring):
470             stdin_file.write(stdin)
471         else:
472             for i in stdin:
473                 stdin_file.write(i + '\n')
474         stdin_file.flush()
475         stdin_file.seek(0)
476
477     p4 = subprocess.Popen(cmd,
478                           shell=expand,
479                           stdin=stdin_file,
480                           stdout=subprocess.PIPE)
481
482     result = []
483     try:
484         while True:
485             entry = marshal.load(p4.stdout)
486             if cb is not None:
487                 cb(entry)
488             else:
489                 result.append(entry)
490     except EOFError:
491         pass
492     exitCode = p4.wait()
493     if exitCode != 0:
494         entry = {}
495         entry["p4ExitCode"] = exitCode
496         result.append(entry)
497
498     return result
499
500 def p4Cmd(cmd):
501     list = p4CmdList(cmd)
502     result = {}
503     for entry in list:
504         result.update(entry)
505     return result;
506
507 def p4Where(depotPath):
508     if not depotPath.endswith("/"):
509         depotPath += "/"
510     depotPathLong = depotPath + "..."
511     outputList = p4CmdList(["where", depotPathLong])
512     output = None
513     for entry in outputList:
514         if "depotFile" in entry:
515             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
516             # The base path always ends with "/...".
517             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
518                 output = entry
519                 break
520         elif "data" in entry:
521             data = entry.get("data")
522             space = data.find(" ")
523             if data[:space] == depotPath:
524                 output = entry
525                 break
526     if output == None:
527         return ""
528     if output["code"] == "error":
529         return ""
530     clientPath = ""
531     if "path" in output:
532         clientPath = output.get("path")
533     elif "data" in output:
534         data = output.get("data")
535         lastSpace = data.rfind(" ")
536         clientPath = data[lastSpace + 1:]
537
538     if clientPath.endswith("..."):
539         clientPath = clientPath[:-3]
540     return clientPath
541
542 def currentGitBranch():
543     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
544
545 def isValidGitDir(path):
546     if (os.path.exists(path + "/HEAD")
547         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
548         return True;
549     return False
550
551 def parseRevision(ref):
552     return read_pipe("git rev-parse %s" % ref).strip()
553
554 def branchExists(ref):
555     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
556                      ignore_error=True)
557     return len(rev) > 0
558
559 def extractLogMessageFromGitCommit(commit):
560     logMessage = ""
561
562     ## fixme: title is first line of commit, not 1st paragraph.
563     foundTitle = False
564     for log in read_pipe_lines("git cat-file commit %s" % commit):
565        if not foundTitle:
566            if len(log) == 1:
567                foundTitle = True
568            continue
569
570        logMessage += log
571     return logMessage
572
573 def extractSettingsGitLog(log):
574     values = {}
575     for line in log.split("\n"):
576         line = line.strip()
577         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
578         if not m:
579             continue
580
581         assignments = m.group(1).split (':')
582         for a in assignments:
583             vals = a.split ('=')
584             key = vals[0].strip()
585             val = ('='.join (vals[1:])).strip()
586             if val.endswith ('\"') and val.startswith('"'):
587                 val = val[1:-1]
588
589             values[key] = val
590
591     paths = values.get("depot-paths")
592     if not paths:
593         paths = values.get("depot-path")
594     if paths:
595         values['depot-paths'] = paths.split(',')
596     return values
597
598 def gitBranchExists(branch):
599     proc = subprocess.Popen(["git", "rev-parse", branch],
600                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
601     return proc.wait() == 0;
602
603 _gitConfig = {}
604
605 def gitConfig(key):
606     if not _gitConfig.has_key(key):
607         cmd = [ "git", "config", key ]
608         s = read_pipe(cmd, ignore_error=True)
609         _gitConfig[key] = s.strip()
610     return _gitConfig[key]
611
612 def gitConfigBool(key):
613     """Return a bool, using git config --bool.  It is True only if the
614        variable is set to true, and False if set to false or not present
615        in the config."""
616
617     if not _gitConfig.has_key(key):
618         cmd = [ "git", "config", "--bool", key ]
619         s = read_pipe(cmd, ignore_error=True)
620         v = s.strip()
621         _gitConfig[key] = v == "true"
622     return _gitConfig[key]
623
624 def gitConfigList(key):
625     if not _gitConfig.has_key(key):
626         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
627         _gitConfig[key] = s.strip().split(os.linesep)
628     return _gitConfig[key]
629
630 def p4BranchesInGit(branchesAreInRemotes=True):
631     """Find all the branches whose names start with "p4/", looking
632        in remotes or heads as specified by the argument.  Return
633        a dictionary of { branch: revision } for each one found.
634        The branch names are the short names, without any
635        "p4/" prefix."""
636
637     branches = {}
638
639     cmdline = "git rev-parse --symbolic "
640     if branchesAreInRemotes:
641         cmdline += "--remotes"
642     else:
643         cmdline += "--branches"
644
645     for line in read_pipe_lines(cmdline):
646         line = line.strip()
647
648         # only import to p4/
649         if not line.startswith('p4/'):
650             continue
651         # special symbolic ref to p4/master
652         if line == "p4/HEAD":
653             continue
654
655         # strip off p4/ prefix
656         branch = line[len("p4/"):]
657
658         branches[branch] = parseRevision(line)
659
660     return branches
661
662 def branch_exists(branch):
663     """Make sure that the given ref name really exists."""
664
665     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
666     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
667     out, _ = p.communicate()
668     if p.returncode:
669         return False
670     # expect exactly one line of output: the branch name
671     return out.rstrip() == branch
672
673 def findUpstreamBranchPoint(head = "HEAD"):
674     branches = p4BranchesInGit()
675     # map from depot-path to branch name
676     branchByDepotPath = {}
677     for branch in branches.keys():
678         tip = branches[branch]
679         log = extractLogMessageFromGitCommit(tip)
680         settings = extractSettingsGitLog(log)
681         if settings.has_key("depot-paths"):
682             paths = ",".join(settings["depot-paths"])
683             branchByDepotPath[paths] = "remotes/p4/" + branch
684
685     settings = None
686     parent = 0
687     while parent < 65535:
688         commit = head + "~%s" % parent
689         log = extractLogMessageFromGitCommit(commit)
690         settings = extractSettingsGitLog(log)
691         if settings.has_key("depot-paths"):
692             paths = ",".join(settings["depot-paths"])
693             if branchByDepotPath.has_key(paths):
694                 return [branchByDepotPath[paths], settings]
695
696         parent = parent + 1
697
698     return ["", settings]
699
700 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
701     if not silent:
702         print ("Creating/updating branch(es) in %s based on origin branch(es)"
703                % localRefPrefix)
704
705     originPrefix = "origin/p4/"
706
707     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
708         line = line.strip()
709         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
710             continue
711
712         headName = line[len(originPrefix):]
713         remoteHead = localRefPrefix + headName
714         originHead = line
715
716         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
717         if (not original.has_key('depot-paths')
718             or not original.has_key('change')):
719             continue
720
721         update = False
722         if not gitBranchExists(remoteHead):
723             if verbose:
724                 print "creating %s" % remoteHead
725             update = True
726         else:
727             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
728             if settings.has_key('change') > 0:
729                 if settings['depot-paths'] == original['depot-paths']:
730                     originP4Change = int(original['change'])
731                     p4Change = int(settings['change'])
732                     if originP4Change > p4Change:
733                         print ("%s (%s) is newer than %s (%s). "
734                                "Updating p4 branch from origin."
735                                % (originHead, originP4Change,
736                                   remoteHead, p4Change))
737                         update = True
738                 else:
739                     print ("Ignoring: %s was imported from %s while "
740                            "%s was imported from %s"
741                            % (originHead, ','.join(original['depot-paths']),
742                               remoteHead, ','.join(settings['depot-paths'])))
743
744         if update:
745             system("git update-ref %s %s" % (remoteHead, originHead))
746
747 def originP4BranchesExist():
748         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
749
750
751 def p4ParseNumericChangeRange(parts):
752     changeStart = int(parts[0][1:])
753     if parts[1] == '#head':
754         changeEnd = p4_last_change()
755     else:
756         changeEnd = int(parts[1])
757
758     return (changeStart, changeEnd)
759
760 def chooseBlockSize(blockSize):
761     if blockSize:
762         return blockSize
763     else:
764         return defaultBlockSize
765
766 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
767     assert depotPaths
768
769     # Parse the change range into start and end. Try to find integer
770     # revision ranges as these can be broken up into blocks to avoid
771     # hitting server-side limits (maxrows, maxscanresults). But if
772     # that doesn't work, fall back to using the raw revision specifier
773     # strings, without using block mode.
774
775     if changeRange is None or changeRange == '':
776         changeStart = 1
777         changeEnd = p4_last_change()
778         block_size = chooseBlockSize(requestedBlockSize)
779     else:
780         parts = changeRange.split(',')
781         assert len(parts) == 2
782         try:
783             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
784             block_size = chooseBlockSize(requestedBlockSize)
785         except:
786             changeStart = parts[0][1:]
787             changeEnd = parts[1]
788             if requestedBlockSize:
789                 die("cannot use --changes-block-size with non-numeric revisions")
790             block_size = None
791
792     # Accumulate change numbers in a dictionary to avoid duplicates
793     changes = {}
794
795     for p in depotPaths:
796         # Retrieve changes a block at a time, to prevent running
797         # into a MaxResults/MaxScanRows error from the server.
798
799         while True:
800             cmd = ['changes']
801
802             if block_size:
803                 end = min(changeEnd, changeStart + block_size)
804                 revisionRange = "%d,%d" % (changeStart, end)
805             else:
806                 revisionRange = "%s,%s" % (changeStart, changeEnd)
807
808             cmd += ["%s...@%s" % (p, revisionRange)]
809
810             for line in p4_read_pipe_lines(cmd):
811                 changeNum = int(line.split(" ")[1])
812                 changes[changeNum] = True
813
814             if not block_size:
815                 break
816
817             if end >= changeEnd:
818                 break
819
820             changeStart = end + 1
821
822     changelist = changes.keys()
823     changelist.sort()
824     return changelist
825
826 def p4PathStartsWith(path, prefix):
827     # This method tries to remedy a potential mixed-case issue:
828     #
829     # If UserA adds  //depot/DirA/file1
830     # and UserB adds //depot/dira/file2
831     #
832     # we may or may not have a problem. If you have core.ignorecase=true,
833     # we treat DirA and dira as the same directory
834     if gitConfigBool("core.ignorecase"):
835         return path.lower().startswith(prefix.lower())
836     return path.startswith(prefix)
837
838 def getClientSpec():
839     """Look at the p4 client spec, create a View() object that contains
840        all the mappings, and return it."""
841
842     specList = p4CmdList("client -o")
843     if len(specList) != 1:
844         die('Output from "client -o" is %d lines, expecting 1' %
845             len(specList))
846
847     # dictionary of all client parameters
848     entry = specList[0]
849
850     # the //client/ name
851     client_name = entry["Client"]
852
853     # just the keys that start with "View"
854     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
855
856     # hold this new View
857     view = View(client_name)
858
859     # append the lines, in order, to the view
860     for view_num in range(len(view_keys)):
861         k = "View%d" % view_num
862         if k not in view_keys:
863             die("Expected view key %s missing" % k)
864         view.append(entry[k])
865
866     return view
867
868 def getClientRoot():
869     """Grab the client directory."""
870
871     output = p4CmdList("client -o")
872     if len(output) != 1:
873         die('Output from "client -o" is %d lines, expecting 1' % len(output))
874
875     entry = output[0]
876     if "Root" not in entry:
877         die('Client has no "Root"')
878
879     return entry["Root"]
880
881 #
882 # P4 wildcards are not allowed in filenames.  P4 complains
883 # if you simply add them, but you can force it with "-f", in
884 # which case it translates them into %xx encoding internally.
885 #
886 def wildcard_decode(path):
887     # Search for and fix just these four characters.  Do % last so
888     # that fixing it does not inadvertently create new %-escapes.
889     # Cannot have * in a filename in windows; untested as to
890     # what p4 would do in such a case.
891     if not platform.system() == "Windows":
892         path = path.replace("%2A", "*")
893     path = path.replace("%23", "#") \
894                .replace("%40", "@") \
895                .replace("%25", "%")
896     return path
897
898 def wildcard_encode(path):
899     # do % first to avoid double-encoding the %s introduced here
900     path = path.replace("%", "%25") \
901                .replace("*", "%2A") \
902                .replace("#", "%23") \
903                .replace("@", "%40")
904     return path
905
906 def wildcard_present(path):
907     m = re.search("[*#@%]", path)
908     return m is not None
909
910 class Command:
911     def __init__(self):
912         self.usage = "usage: %prog [options]"
913         self.needsGit = True
914         self.verbose = False
915
916 class P4UserMap:
917     def __init__(self):
918         self.userMapFromPerforceServer = False
919         self.myP4UserId = None
920
921     def p4UserId(self):
922         if self.myP4UserId:
923             return self.myP4UserId
924
925         results = p4CmdList("user -o")
926         for r in results:
927             if r.has_key('User'):
928                 self.myP4UserId = r['User']
929                 return r['User']
930         die("Could not find your p4 user id")
931
932     def p4UserIsMe(self, p4User):
933         # return True if the given p4 user is actually me
934         me = self.p4UserId()
935         if not p4User or p4User != me:
936             return False
937         else:
938             return True
939
940     def getUserCacheFilename(self):
941         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
942         return home + "/.gitp4-usercache.txt"
943
944     def getUserMapFromPerforceServer(self):
945         if self.userMapFromPerforceServer:
946             return
947         self.users = {}
948         self.emails = {}
949
950         for output in p4CmdList("users"):
951             if not output.has_key("User"):
952                 continue
953             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
954             self.emails[output["Email"]] = output["User"]
955
956
957         s = ''
958         for (key, val) in self.users.items():
959             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
960
961         open(self.getUserCacheFilename(), "wb").write(s)
962         self.userMapFromPerforceServer = True
963
964     def loadUserMapFromCache(self):
965         self.users = {}
966         self.userMapFromPerforceServer = False
967         try:
968             cache = open(self.getUserCacheFilename(), "rb")
969             lines = cache.readlines()
970             cache.close()
971             for line in lines:
972                 entry = line.strip().split("\t")
973                 self.users[entry[0]] = entry[1]
974         except IOError:
975             self.getUserMapFromPerforceServer()
976
977 class P4Debug(Command):
978     def __init__(self):
979         Command.__init__(self)
980         self.options = []
981         self.description = "A tool to debug the output of p4 -G."
982         self.needsGit = False
983
984     def run(self, args):
985         j = 0
986         for output in p4CmdList(args):
987             print 'Element: %d' % j
988             j += 1
989             print output
990         return True
991
992 class P4RollBack(Command):
993     def __init__(self):
994         Command.__init__(self)
995         self.options = [
996             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
997         ]
998         self.description = "A tool to debug the multi-branch import. Don't use :)"
999         self.rollbackLocalBranches = False
1000
1001     def run(self, args):
1002         if len(args) != 1:
1003             return False
1004         maxChange = int(args[0])
1005
1006         if "p4ExitCode" in p4Cmd("changes -m 1"):
1007             die("Problems executing p4");
1008
1009         if self.rollbackLocalBranches:
1010             refPrefix = "refs/heads/"
1011             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1012         else:
1013             refPrefix = "refs/remotes/"
1014             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1015
1016         for line in lines:
1017             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1018                 line = line.strip()
1019                 ref = refPrefix + line
1020                 log = extractLogMessageFromGitCommit(ref)
1021                 settings = extractSettingsGitLog(log)
1022
1023                 depotPaths = settings['depot-paths']
1024                 change = settings['change']
1025
1026                 changed = False
1027
1028                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1029                                                            for p in depotPaths]))) == 0:
1030                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1031                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1032                     continue
1033
1034                 while change and int(change) > maxChange:
1035                     changed = True
1036                     if self.verbose:
1037                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1038                     system("git update-ref %s \"%s^\"" % (ref, ref))
1039                     log = extractLogMessageFromGitCommit(ref)
1040                     settings =  extractSettingsGitLog(log)
1041
1042
1043                     depotPaths = settings['depot-paths']
1044                     change = settings['change']
1045
1046                 if changed:
1047                     print "%s rewound to %s" % (ref, change)
1048
1049         return True
1050
1051 class P4Submit(Command, P4UserMap):
1052
1053     conflict_behavior_choices = ("ask", "skip", "quit")
1054
1055     def __init__(self):
1056         Command.__init__(self)
1057         P4UserMap.__init__(self)
1058         self.options = [
1059                 optparse.make_option("--origin", dest="origin"),
1060                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1061                 # preserve the user, requires relevant p4 permissions
1062                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1063                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1064                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1065                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1066                 optparse.make_option("--conflict", dest="conflict_behavior",
1067                                      choices=self.conflict_behavior_choices),
1068                 optparse.make_option("--branch", dest="branch"),
1069         ]
1070         self.description = "Submit changes from git to the perforce depot."
1071         self.usage += " [name of git branch to submit into perforce depot]"
1072         self.origin = ""
1073         self.detectRenames = False
1074         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1075         self.dry_run = False
1076         self.prepare_p4_only = False
1077         self.conflict_behavior = None
1078         self.isWindows = (platform.system() == "Windows")
1079         self.exportLabels = False
1080         self.p4HasMoveCommand = p4_has_move_command()
1081         self.branch = None
1082
1083     def check(self):
1084         if len(p4CmdList("opened ...")) > 0:
1085             die("You have files opened with perforce! Close them before starting the sync.")
1086
1087     def separate_jobs_from_description(self, message):
1088         """Extract and return a possible Jobs field in the commit
1089            message.  It goes into a separate section in the p4 change
1090            specification.
1091
1092            A jobs line starts with "Jobs:" and looks like a new field
1093            in a form.  Values are white-space separated on the same
1094            line or on following lines that start with a tab.
1095
1096            This does not parse and extract the full git commit message
1097            like a p4 form.  It just sees the Jobs: line as a marker
1098            to pass everything from then on directly into the p4 form,
1099            but outside the description section.
1100
1101            Return a tuple (stripped log message, jobs string)."""
1102
1103         m = re.search(r'^Jobs:', message, re.MULTILINE)
1104         if m is None:
1105             return (message, None)
1106
1107         jobtext = message[m.start():]
1108         stripped_message = message[:m.start()].rstrip()
1109         return (stripped_message, jobtext)
1110
1111     def prepareLogMessage(self, template, message, jobs):
1112         """Edits the template returned from "p4 change -o" to insert
1113            the message in the Description field, and the jobs text in
1114            the Jobs field."""
1115         result = ""
1116
1117         inDescriptionSection = False
1118
1119         for line in template.split("\n"):
1120             if line.startswith("#"):
1121                 result += line + "\n"
1122                 continue
1123
1124             if inDescriptionSection:
1125                 if line.startswith("Files:") or line.startswith("Jobs:"):
1126                     inDescriptionSection = False
1127                     # insert Jobs section
1128                     if jobs:
1129                         result += jobs + "\n"
1130                 else:
1131                     continue
1132             else:
1133                 if line.startswith("Description:"):
1134                     inDescriptionSection = True
1135                     line += "\n"
1136                     for messageLine in message.split("\n"):
1137                         line += "\t" + messageLine + "\n"
1138
1139             result += line + "\n"
1140
1141         return result
1142
1143     def patchRCSKeywords(self, file, pattern):
1144         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1145         (handle, outFileName) = tempfile.mkstemp(dir='.')
1146         try:
1147             outFile = os.fdopen(handle, "w+")
1148             inFile = open(file, "r")
1149             regexp = re.compile(pattern, re.VERBOSE)
1150             for line in inFile.readlines():
1151                 line = regexp.sub(r'$\1$', line)
1152                 outFile.write(line)
1153             inFile.close()
1154             outFile.close()
1155             # Forcibly overwrite the original file
1156             os.unlink(file)
1157             shutil.move(outFileName, file)
1158         except:
1159             # cleanup our temporary file
1160             os.unlink(outFileName)
1161             print "Failed to strip RCS keywords in %s" % file
1162             raise
1163
1164         print "Patched up RCS keywords in %s" % file
1165
1166     def p4UserForCommit(self,id):
1167         # Return the tuple (perforce user,git email) for a given git commit id
1168         self.getUserMapFromPerforceServer()
1169         gitEmail = read_pipe(["git", "log", "--max-count=1",
1170                               "--format=%ae", id])
1171         gitEmail = gitEmail.strip()
1172         if not self.emails.has_key(gitEmail):
1173             return (None,gitEmail)
1174         else:
1175             return (self.emails[gitEmail],gitEmail)
1176
1177     def checkValidP4Users(self,commits):
1178         # check if any git authors cannot be mapped to p4 users
1179         for id in commits:
1180             (user,email) = self.p4UserForCommit(id)
1181             if not user:
1182                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1183                 if gitConfigBool("git-p4.allowMissingP4Users"):
1184                     print "%s" % msg
1185                 else:
1186                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1187
1188     def lastP4Changelist(self):
1189         # Get back the last changelist number submitted in this client spec. This
1190         # then gets used to patch up the username in the change. If the same
1191         # client spec is being used by multiple processes then this might go
1192         # wrong.
1193         results = p4CmdList("client -o")        # find the current client
1194         client = None
1195         for r in results:
1196             if r.has_key('Client'):
1197                 client = r['Client']
1198                 break
1199         if not client:
1200             die("could not get client spec")
1201         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1202         for r in results:
1203             if r.has_key('change'):
1204                 return r['change']
1205         die("Could not get changelist number for last submit - cannot patch up user details")
1206
1207     def modifyChangelistUser(self, changelist, newUser):
1208         # fixup the user field of a changelist after it has been submitted.
1209         changes = p4CmdList("change -o %s" % changelist)
1210         if len(changes) != 1:
1211             die("Bad output from p4 change modifying %s to user %s" %
1212                 (changelist, newUser))
1213
1214         c = changes[0]
1215         if c['User'] == newUser: return   # nothing to do
1216         c['User'] = newUser
1217         input = marshal.dumps(c)
1218
1219         result = p4CmdList("change -f -i", stdin=input)
1220         for r in result:
1221             if r.has_key('code'):
1222                 if r['code'] == 'error':
1223                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1224             if r.has_key('data'):
1225                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1226                 return
1227         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1228
1229     def canChangeChangelists(self):
1230         # check to see if we have p4 admin or super-user permissions, either of
1231         # which are required to modify changelists.
1232         results = p4CmdList(["protects", self.depotPath])
1233         for r in results:
1234             if r.has_key('perm'):
1235                 if r['perm'] == 'admin':
1236                     return 1
1237                 if r['perm'] == 'super':
1238                     return 1
1239         return 0
1240
1241     def prepareSubmitTemplate(self):
1242         """Run "p4 change -o" to grab a change specification template.
1243            This does not use "p4 -G", as it is nice to keep the submission
1244            template in original order, since a human might edit it.
1245
1246            Remove lines in the Files section that show changes to files
1247            outside the depot path we're committing into."""
1248
1249         template = ""
1250         inFilesSection = False
1251         for line in p4_read_pipe_lines(['change', '-o']):
1252             if line.endswith("\r\n"):
1253                 line = line[:-2] + "\n"
1254             if inFilesSection:
1255                 if line.startswith("\t"):
1256                     # path starts and ends with a tab
1257                     path = line[1:]
1258                     lastTab = path.rfind("\t")
1259                     if lastTab != -1:
1260                         path = path[:lastTab]
1261                         if not p4PathStartsWith(path, self.depotPath):
1262                             continue
1263                 else:
1264                     inFilesSection = False
1265             else:
1266                 if line.startswith("Files:"):
1267                     inFilesSection = True
1268
1269             template += line
1270
1271         return template
1272
1273     def edit_template(self, template_file):
1274         """Invoke the editor to let the user change the submission
1275            message.  Return true if okay to continue with the submit."""
1276
1277         # if configured to skip the editing part, just submit
1278         if gitConfigBool("git-p4.skipSubmitEdit"):
1279             return True
1280
1281         # look at the modification time, to check later if the user saved
1282         # the file
1283         mtime = os.stat(template_file).st_mtime
1284
1285         # invoke the editor
1286         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1287             editor = os.environ.get("P4EDITOR")
1288         else:
1289             editor = read_pipe("git var GIT_EDITOR").strip()
1290         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1291
1292         # If the file was not saved, prompt to see if this patch should
1293         # be skipped.  But skip this verification step if configured so.
1294         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1295             return True
1296
1297         # modification time updated means user saved the file
1298         if os.stat(template_file).st_mtime > mtime:
1299             return True
1300
1301         while True:
1302             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1303             if response == 'y':
1304                 return True
1305             if response == 'n':
1306                 return False
1307
1308     def get_diff_description(self, editedFiles, filesToAdd):
1309         # diff
1310         if os.environ.has_key("P4DIFF"):
1311             del(os.environ["P4DIFF"])
1312         diff = ""
1313         for editedFile in editedFiles:
1314             diff += p4_read_pipe(['diff', '-du',
1315                                   wildcard_encode(editedFile)])
1316
1317         # new file diff
1318         newdiff = ""
1319         for newFile in filesToAdd:
1320             newdiff += "==== new file ====\n"
1321             newdiff += "--- /dev/null\n"
1322             newdiff += "+++ %s\n" % newFile
1323             f = open(newFile, "r")
1324             for line in f.readlines():
1325                 newdiff += "+" + line
1326             f.close()
1327
1328         return (diff + newdiff).replace('\r\n', '\n')
1329
1330     def applyCommit(self, id):
1331         """Apply one commit, return True if it succeeded."""
1332
1333         print "Applying", read_pipe(["git", "show", "-s",
1334                                      "--format=format:%h %s", id])
1335
1336         (p4User, gitEmail) = self.p4UserForCommit(id)
1337
1338         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1339         filesToAdd = set()
1340         filesToDelete = set()
1341         editedFiles = set()
1342         pureRenameCopy = set()
1343         filesToChangeExecBit = {}
1344
1345         for line in diff:
1346             diff = parseDiffTreeEntry(line)
1347             modifier = diff['status']
1348             path = diff['src']
1349             if modifier == "M":
1350                 p4_edit(path)
1351                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1352                     filesToChangeExecBit[path] = diff['dst_mode']
1353                 editedFiles.add(path)
1354             elif modifier == "A":
1355                 filesToAdd.add(path)
1356                 filesToChangeExecBit[path] = diff['dst_mode']
1357                 if path in filesToDelete:
1358                     filesToDelete.remove(path)
1359             elif modifier == "D":
1360                 filesToDelete.add(path)
1361                 if path in filesToAdd:
1362                     filesToAdd.remove(path)
1363             elif modifier == "C":
1364                 src, dest = diff['src'], diff['dst']
1365                 p4_integrate(src, dest)
1366                 pureRenameCopy.add(dest)
1367                 if diff['src_sha1'] != diff['dst_sha1']:
1368                     p4_edit(dest)
1369                     pureRenameCopy.discard(dest)
1370                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1371                     p4_edit(dest)
1372                     pureRenameCopy.discard(dest)
1373                     filesToChangeExecBit[dest] = diff['dst_mode']
1374                 if self.isWindows:
1375                     # turn off read-only attribute
1376                     os.chmod(dest, stat.S_IWRITE)
1377                 os.unlink(dest)
1378                 editedFiles.add(dest)
1379             elif modifier == "R":
1380                 src, dest = diff['src'], diff['dst']
1381                 if self.p4HasMoveCommand:
1382                     p4_edit(src)        # src must be open before move
1383                     p4_move(src, dest)  # opens for (move/delete, move/add)
1384                 else:
1385                     p4_integrate(src, dest)
1386                     if diff['src_sha1'] != diff['dst_sha1']:
1387                         p4_edit(dest)
1388                     else:
1389                         pureRenameCopy.add(dest)
1390                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1391                     if not self.p4HasMoveCommand:
1392                         p4_edit(dest)   # with move: already open, writable
1393                     filesToChangeExecBit[dest] = diff['dst_mode']
1394                 if not self.p4HasMoveCommand:
1395                     if self.isWindows:
1396                         os.chmod(dest, stat.S_IWRITE)
1397                     os.unlink(dest)
1398                     filesToDelete.add(src)
1399                 editedFiles.add(dest)
1400             else:
1401                 die("unknown modifier %s for %s" % (modifier, path))
1402
1403         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1404         patchcmd = diffcmd + " | git apply "
1405         tryPatchCmd = patchcmd + "--check -"
1406         applyPatchCmd = patchcmd + "--check --apply -"
1407         patch_succeeded = True
1408
1409         if os.system(tryPatchCmd) != 0:
1410             fixed_rcs_keywords = False
1411             patch_succeeded = False
1412             print "Unfortunately applying the change failed!"
1413
1414             # Patch failed, maybe it's just RCS keyword woes. Look through
1415             # the patch to see if that's possible.
1416             if gitConfigBool("git-p4.attemptRCSCleanup"):
1417                 file = None
1418                 pattern = None
1419                 kwfiles = {}
1420                 for file in editedFiles | filesToDelete:
1421                     # did this file's delta contain RCS keywords?
1422                     pattern = p4_keywords_regexp_for_file(file)
1423
1424                     if pattern:
1425                         # this file is a possibility...look for RCS keywords.
1426                         regexp = re.compile(pattern, re.VERBOSE)
1427                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1428                             if regexp.search(line):
1429                                 if verbose:
1430                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1431                                 kwfiles[file] = pattern
1432                                 break
1433
1434                 for file in kwfiles:
1435                     if verbose:
1436                         print "zapping %s with %s" % (line,pattern)
1437                     # File is being deleted, so not open in p4.  Must
1438                     # disable the read-only bit on windows.
1439                     if self.isWindows and file not in editedFiles:
1440                         os.chmod(file, stat.S_IWRITE)
1441                     self.patchRCSKeywords(file, kwfiles[file])
1442                     fixed_rcs_keywords = True
1443
1444             if fixed_rcs_keywords:
1445                 print "Retrying the patch with RCS keywords cleaned up"
1446                 if os.system(tryPatchCmd) == 0:
1447                     patch_succeeded = True
1448
1449         if not patch_succeeded:
1450             for f in editedFiles:
1451                 p4_revert(f)
1452             return False
1453
1454         #
1455         # Apply the patch for real, and do add/delete/+x handling.
1456         #
1457         system(applyPatchCmd)
1458
1459         for f in filesToAdd:
1460             p4_add(f)
1461         for f in filesToDelete:
1462             p4_revert(f)
1463             p4_delete(f)
1464
1465         # Set/clear executable bits
1466         for f in filesToChangeExecBit.keys():
1467             mode = filesToChangeExecBit[f]
1468             setP4ExecBit(f, mode)
1469
1470         #
1471         # Build p4 change description, starting with the contents
1472         # of the git commit message.
1473         #
1474         logMessage = extractLogMessageFromGitCommit(id)
1475         logMessage = logMessage.strip()
1476         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1477
1478         template = self.prepareSubmitTemplate()
1479         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1480
1481         if self.preserveUser:
1482            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1483
1484         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1485             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1486             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1487             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1488
1489         separatorLine = "######## everything below this line is just the diff #######\n"
1490         if not self.prepare_p4_only:
1491             submitTemplate += separatorLine
1492             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1493
1494         (handle, fileName) = tempfile.mkstemp()
1495         tmpFile = os.fdopen(handle, "w+b")
1496         if self.isWindows:
1497             submitTemplate = submitTemplate.replace("\n", "\r\n")
1498         tmpFile.write(submitTemplate)
1499         tmpFile.close()
1500
1501         if self.prepare_p4_only:
1502             #
1503             # Leave the p4 tree prepared, and the submit template around
1504             # and let the user decide what to do next
1505             #
1506             print
1507             print "P4 workspace prepared for submission."
1508             print "To submit or revert, go to client workspace"
1509             print "  " + self.clientPath
1510             print
1511             print "To submit, use \"p4 submit\" to write a new description,"
1512             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1513                   " \"git p4\"." % fileName
1514             print "You can delete the file \"%s\" when finished." % fileName
1515
1516             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1517                 print "To preserve change ownership by user %s, you must\n" \
1518                       "do \"p4 change -f <change>\" after submitting and\n" \
1519                       "edit the User field."
1520             if pureRenameCopy:
1521                 print "After submitting, renamed files must be re-synced."
1522                 print "Invoke \"p4 sync -f\" on each of these files:"
1523                 for f in pureRenameCopy:
1524                     print "  " + f
1525
1526             print
1527             print "To revert the changes, use \"p4 revert ...\", and delete"
1528             print "the submit template file \"%s\"" % fileName
1529             if filesToAdd:
1530                 print "Since the commit adds new files, they must be deleted:"
1531                 for f in filesToAdd:
1532                     print "  " + f
1533             print
1534             return True
1535
1536         #
1537         # Let the user edit the change description, then submit it.
1538         #
1539         if self.edit_template(fileName):
1540             # read the edited message and submit
1541             ret = True
1542             tmpFile = open(fileName, "rb")
1543             message = tmpFile.read()
1544             tmpFile.close()
1545             if self.isWindows:
1546                 message = message.replace("\r\n", "\n")
1547             submitTemplate = message[:message.index(separatorLine)]
1548             p4_write_pipe(['submit', '-i'], submitTemplate)
1549
1550             if self.preserveUser:
1551                 if p4User:
1552                     # Get last changelist number. Cannot easily get it from
1553                     # the submit command output as the output is
1554                     # unmarshalled.
1555                     changelist = self.lastP4Changelist()
1556                     self.modifyChangelistUser(changelist, p4User)
1557
1558             # The rename/copy happened by applying a patch that created a
1559             # new file.  This leaves it writable, which confuses p4.
1560             for f in pureRenameCopy:
1561                 p4_sync(f, "-f")
1562
1563         else:
1564             # skip this patch
1565             ret = False
1566             print "Submission cancelled, undoing p4 changes."
1567             for f in editedFiles:
1568                 p4_revert(f)
1569             for f in filesToAdd:
1570                 p4_revert(f)
1571                 os.remove(f)
1572             for f in filesToDelete:
1573                 p4_revert(f)
1574
1575         os.remove(fileName)
1576         return ret
1577
1578     # Export git tags as p4 labels. Create a p4 label and then tag
1579     # with that.
1580     def exportGitTags(self, gitTags):
1581         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1582         if len(validLabelRegexp) == 0:
1583             validLabelRegexp = defaultLabelRegexp
1584         m = re.compile(validLabelRegexp)
1585
1586         for name in gitTags:
1587
1588             if not m.match(name):
1589                 if verbose:
1590                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1591                 continue
1592
1593             # Get the p4 commit this corresponds to
1594             logMessage = extractLogMessageFromGitCommit(name)
1595             values = extractSettingsGitLog(logMessage)
1596
1597             if not values.has_key('change'):
1598                 # a tag pointing to something not sent to p4; ignore
1599                 if verbose:
1600                     print "git tag %s does not give a p4 commit" % name
1601                 continue
1602             else:
1603                 changelist = values['change']
1604
1605             # Get the tag details.
1606             inHeader = True
1607             isAnnotated = False
1608             body = []
1609             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1610                 l = l.strip()
1611                 if inHeader:
1612                     if re.match(r'tag\s+', l):
1613                         isAnnotated = True
1614                     elif re.match(r'\s*$', l):
1615                         inHeader = False
1616                         continue
1617                 else:
1618                     body.append(l)
1619
1620             if not isAnnotated:
1621                 body = ["lightweight tag imported by git p4\n"]
1622
1623             # Create the label - use the same view as the client spec we are using
1624             clientSpec = getClientSpec()
1625
1626             labelTemplate  = "Label: %s\n" % name
1627             labelTemplate += "Description:\n"
1628             for b in body:
1629                 labelTemplate += "\t" + b + "\n"
1630             labelTemplate += "View:\n"
1631             for depot_side in clientSpec.mappings:
1632                 labelTemplate += "\t%s\n" % depot_side
1633
1634             if self.dry_run:
1635                 print "Would create p4 label %s for tag" % name
1636             elif self.prepare_p4_only:
1637                 print "Not creating p4 label %s for tag due to option" \
1638                       " --prepare-p4-only" % name
1639             else:
1640                 p4_write_pipe(["label", "-i"], labelTemplate)
1641
1642                 # Use the label
1643                 p4_system(["tag", "-l", name] +
1644                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1645
1646                 if verbose:
1647                     print "created p4 label for tag %s" % name
1648
1649     def run(self, args):
1650         if len(args) == 0:
1651             self.master = currentGitBranch()
1652             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1653                 die("Detecting current git branch failed!")
1654         elif len(args) == 1:
1655             self.master = args[0]
1656             if not branchExists(self.master):
1657                 die("Branch %s does not exist" % self.master)
1658         else:
1659             return False
1660
1661         allowSubmit = gitConfig("git-p4.allowSubmit")
1662         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1663             die("%s is not in git-p4.allowSubmit" % self.master)
1664
1665         [upstream, settings] = findUpstreamBranchPoint()
1666         self.depotPath = settings['depot-paths'][0]
1667         if len(self.origin) == 0:
1668             self.origin = upstream
1669
1670         if self.preserveUser:
1671             if not self.canChangeChangelists():
1672                 die("Cannot preserve user names without p4 super-user or admin permissions")
1673
1674         # if not set from the command line, try the config file
1675         if self.conflict_behavior is None:
1676             val = gitConfig("git-p4.conflict")
1677             if val:
1678                 if val not in self.conflict_behavior_choices:
1679                     die("Invalid value '%s' for config git-p4.conflict" % val)
1680             else:
1681                 val = "ask"
1682             self.conflict_behavior = val
1683
1684         if self.verbose:
1685             print "Origin branch is " + self.origin
1686
1687         if len(self.depotPath) == 0:
1688             print "Internal error: cannot locate perforce depot path from existing branches"
1689             sys.exit(128)
1690
1691         self.useClientSpec = False
1692         if gitConfigBool("git-p4.useclientspec"):
1693             self.useClientSpec = True
1694         if self.useClientSpec:
1695             self.clientSpecDirs = getClientSpec()
1696
1697         # Check for the existance of P4 branches
1698         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1699
1700         if self.useClientSpec and not branchesDetected:
1701             # all files are relative to the client spec
1702             self.clientPath = getClientRoot()
1703         else:
1704             self.clientPath = p4Where(self.depotPath)
1705
1706         if self.clientPath == "":
1707             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1708
1709         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1710         self.oldWorkingDirectory = os.getcwd()
1711
1712         # ensure the clientPath exists
1713         new_client_dir = False
1714         if not os.path.exists(self.clientPath):
1715             new_client_dir = True
1716             os.makedirs(self.clientPath)
1717
1718         chdir(self.clientPath, is_client_path=True)
1719         if self.dry_run:
1720             print "Would synchronize p4 checkout in %s" % self.clientPath
1721         else:
1722             print "Synchronizing p4 checkout..."
1723             if new_client_dir:
1724                 # old one was destroyed, and maybe nobody told p4
1725                 p4_sync("...", "-f")
1726             else:
1727                 p4_sync("...")
1728         self.check()
1729
1730         commits = []
1731         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1732             commits.append(line.strip())
1733         commits.reverse()
1734
1735         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1736             self.checkAuthorship = False
1737         else:
1738             self.checkAuthorship = True
1739
1740         if self.preserveUser:
1741             self.checkValidP4Users(commits)
1742
1743         #
1744         # Build up a set of options to be passed to diff when
1745         # submitting each commit to p4.
1746         #
1747         if self.detectRenames:
1748             # command-line -M arg
1749             self.diffOpts = "-M"
1750         else:
1751             # If not explicitly set check the config variable
1752             detectRenames = gitConfig("git-p4.detectRenames")
1753
1754             if detectRenames.lower() == "false" or detectRenames == "":
1755                 self.diffOpts = ""
1756             elif detectRenames.lower() == "true":
1757                 self.diffOpts = "-M"
1758             else:
1759                 self.diffOpts = "-M%s" % detectRenames
1760
1761         # no command-line arg for -C or --find-copies-harder, just
1762         # config variables
1763         detectCopies = gitConfig("git-p4.detectCopies")
1764         if detectCopies.lower() == "false" or detectCopies == "":
1765             pass
1766         elif detectCopies.lower() == "true":
1767             self.diffOpts += " -C"
1768         else:
1769             self.diffOpts += " -C%s" % detectCopies
1770
1771         if gitConfigBool("git-p4.detectCopiesHarder"):
1772             self.diffOpts += " --find-copies-harder"
1773
1774         #
1775         # Apply the commits, one at a time.  On failure, ask if should
1776         # continue to try the rest of the patches, or quit.
1777         #
1778         if self.dry_run:
1779             print "Would apply"
1780         applied = []
1781         last = len(commits) - 1
1782         for i, commit in enumerate(commits):
1783             if self.dry_run:
1784                 print " ", read_pipe(["git", "show", "-s",
1785                                       "--format=format:%h %s", commit])
1786                 ok = True
1787             else:
1788                 ok = self.applyCommit(commit)
1789             if ok:
1790                 applied.append(commit)
1791             else:
1792                 if self.prepare_p4_only and i < last:
1793                     print "Processing only the first commit due to option" \
1794                           " --prepare-p4-only"
1795                     break
1796                 if i < last:
1797                     quit = False
1798                     while True:
1799                         # prompt for what to do, or use the option/variable
1800                         if self.conflict_behavior == "ask":
1801                             print "What do you want to do?"
1802                             response = raw_input("[s]kip this commit but apply"
1803                                                  " the rest, or [q]uit? ")
1804                             if not response:
1805                                 continue
1806                         elif self.conflict_behavior == "skip":
1807                             response = "s"
1808                         elif self.conflict_behavior == "quit":
1809                             response = "q"
1810                         else:
1811                             die("Unknown conflict_behavior '%s'" %
1812                                 self.conflict_behavior)
1813
1814                         if response[0] == "s":
1815                             print "Skipping this commit, but applying the rest"
1816                             break
1817                         if response[0] == "q":
1818                             print "Quitting"
1819                             quit = True
1820                             break
1821                     if quit:
1822                         break
1823
1824         chdir(self.oldWorkingDirectory)
1825
1826         if self.dry_run:
1827             pass
1828         elif self.prepare_p4_only:
1829             pass
1830         elif len(commits) == len(applied):
1831             print "All commits applied!"
1832
1833             sync = P4Sync()
1834             if self.branch:
1835                 sync.branch = self.branch
1836             sync.run([])
1837
1838             rebase = P4Rebase()
1839             rebase.rebase()
1840
1841         else:
1842             if len(applied) == 0:
1843                 print "No commits applied."
1844             else:
1845                 print "Applied only the commits marked with '*':"
1846                 for c in commits:
1847                     if c in applied:
1848                         star = "*"
1849                     else:
1850                         star = " "
1851                     print star, read_pipe(["git", "show", "-s",
1852                                            "--format=format:%h %s",  c])
1853                 print "You will have to do 'git p4 sync' and rebase."
1854
1855         if gitConfigBool("git-p4.exportLabels"):
1856             self.exportLabels = True
1857
1858         if self.exportLabels:
1859             p4Labels = getP4Labels(self.depotPath)
1860             gitTags = getGitTags()
1861
1862             missingGitTags = gitTags - p4Labels
1863             self.exportGitTags(missingGitTags)
1864
1865         # exit with error unless everything applied perfectly
1866         if len(commits) != len(applied):
1867                 sys.exit(1)
1868
1869         return True
1870
1871 class View(object):
1872     """Represent a p4 view ("p4 help views"), and map files in a
1873        repo according to the view."""
1874
1875     def __init__(self, client_name):
1876         self.mappings = []
1877         self.client_prefix = "//%s/" % client_name
1878         # cache results of "p4 where" to lookup client file locations
1879         self.client_spec_path_cache = {}
1880
1881     def append(self, view_line):
1882         """Parse a view line, splitting it into depot and client
1883            sides.  Append to self.mappings, preserving order.  This
1884            is only needed for tag creation."""
1885
1886         # Split the view line into exactly two words.  P4 enforces
1887         # structure on these lines that simplifies this quite a bit.
1888         #
1889         # Either or both words may be double-quoted.
1890         # Single quotes do not matter.
1891         # Double-quote marks cannot occur inside the words.
1892         # A + or - prefix is also inside the quotes.
1893         # There are no quotes unless they contain a space.
1894         # The line is already white-space stripped.
1895         # The two words are separated by a single space.
1896         #
1897         if view_line[0] == '"':
1898             # First word is double quoted.  Find its end.
1899             close_quote_index = view_line.find('"', 1)
1900             if close_quote_index <= 0:
1901                 die("No first-word closing quote found: %s" % view_line)
1902             depot_side = view_line[1:close_quote_index]
1903             # skip closing quote and space
1904             rhs_index = close_quote_index + 1 + 1
1905         else:
1906             space_index = view_line.find(" ")
1907             if space_index <= 0:
1908                 die("No word-splitting space found: %s" % view_line)
1909             depot_side = view_line[0:space_index]
1910             rhs_index = space_index + 1
1911
1912         # prefix + means overlay on previous mapping
1913         if depot_side.startswith("+"):
1914             depot_side = depot_side[1:]
1915
1916         # prefix - means exclude this path, leave out of mappings
1917         exclude = False
1918         if depot_side.startswith("-"):
1919             exclude = True
1920             depot_side = depot_side[1:]
1921
1922         if not exclude:
1923             self.mappings.append(depot_side)
1924
1925     def convert_client_path(self, clientFile):
1926         # chop off //client/ part to make it relative
1927         if not clientFile.startswith(self.client_prefix):
1928             die("No prefix '%s' on clientFile '%s'" %
1929                 (self.client_prefix, clientFile))
1930         return clientFile[len(self.client_prefix):]
1931
1932     def update_client_spec_path_cache(self, files):
1933         """ Caching file paths by "p4 where" batch query """
1934
1935         # List depot file paths exclude that already cached
1936         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1937
1938         if len(fileArgs) == 0:
1939             return  # All files in cache
1940
1941         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1942         for res in where_result:
1943             if "code" in res and res["code"] == "error":
1944                 # assume error is "... file(s) not in client view"
1945                 continue
1946             if "clientFile" not in res:
1947                 die("No clientFile in 'p4 where' output")
1948             if "unmap" in res:
1949                 # it will list all of them, but only one not unmap-ped
1950                 continue
1951             if gitConfigBool("core.ignorecase"):
1952                 res['depotFile'] = res['depotFile'].lower()
1953             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1954
1955         # not found files or unmap files set to ""
1956         for depotFile in fileArgs:
1957             if gitConfigBool("core.ignorecase"):
1958                 depotFile = depotFile.lower()
1959             if depotFile not in self.client_spec_path_cache:
1960                 self.client_spec_path_cache[depotFile] = ""
1961
1962     def map_in_client(self, depot_path):
1963         """Return the relative location in the client where this
1964            depot file should live.  Returns "" if the file should
1965            not be mapped in the client."""
1966
1967         if gitConfigBool("core.ignorecase"):
1968             depot_path = depot_path.lower()
1969
1970         if depot_path in self.client_spec_path_cache:
1971             return self.client_spec_path_cache[depot_path]
1972
1973         die( "Error: %s is not found in client spec path" % depot_path )
1974         return ""
1975
1976 class P4Sync(Command, P4UserMap):
1977     delete_actions = ( "delete", "move/delete", "purge" )
1978
1979     def __init__(self):
1980         Command.__init__(self)
1981         P4UserMap.__init__(self)
1982         self.options = [
1983                 optparse.make_option("--branch", dest="branch"),
1984                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1985                 optparse.make_option("--changesfile", dest="changesFile"),
1986                 optparse.make_option("--silent", dest="silent", action="store_true"),
1987                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1988                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1989                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1990                                      help="Import into refs/heads/ , not refs/remotes"),
1991                 optparse.make_option("--max-changes", dest="maxChanges",
1992                                      help="Maximum number of changes to import"),
1993                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1994                                      help="Internal block size to use when iteratively calling p4 changes"),
1995                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1996                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1997                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1998                                      help="Only sync files that are included in the Perforce Client Spec"),
1999                 optparse.make_option("-/", dest="cloneExclude",
2000                                      action="append", type="string",
2001                                      help="exclude depot path"),
2002         ]
2003         self.description = """Imports from Perforce into a git repository.\n
2004     example:
2005     //depot/my/project/ -- to import the current head
2006     //depot/my/project/@all -- to import everything
2007     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2008
2009     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2010
2011         self.usage += " //depot/path[@revRange]"
2012         self.silent = False
2013         self.createdBranches = set()
2014         self.committedChanges = set()
2015         self.branch = ""
2016         self.detectBranches = False
2017         self.detectLabels = False
2018         self.importLabels = False
2019         self.changesFile = ""
2020         self.syncWithOrigin = True
2021         self.importIntoRemotes = True
2022         self.maxChanges = ""
2023         self.changes_block_size = None
2024         self.keepRepoPath = False
2025         self.depotPaths = None
2026         self.p4BranchesInGit = []
2027         self.cloneExclude = []
2028         self.useClientSpec = False
2029         self.useClientSpec_from_options = False
2030         self.clientSpecDirs = None
2031         self.tempBranches = []
2032         self.tempBranchLocation = "git-p4-tmp"
2033
2034         if gitConfig("git-p4.syncFromOrigin") == "false":
2035             self.syncWithOrigin = False
2036
2037     # This is required for the "append" cloneExclude action
2038     def ensure_value(self, attr, value):
2039         if not hasattr(self, attr) or getattr(self, attr) is None:
2040             setattr(self, attr, value)
2041         return getattr(self, attr)
2042
2043     # Force a checkpoint in fast-import and wait for it to finish
2044     def checkpoint(self):
2045         self.gitStream.write("checkpoint\n\n")
2046         self.gitStream.write("progress checkpoint\n\n")
2047         out = self.gitOutput.readline()
2048         if self.verbose:
2049             print "checkpoint finished: " + out
2050
2051     def extractFilesFromCommit(self, commit):
2052         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2053                              for path in self.cloneExclude]
2054         files = []
2055         fnum = 0
2056         while commit.has_key("depotFile%s" % fnum):
2057             path =  commit["depotFile%s" % fnum]
2058
2059             if [p for p in self.cloneExclude
2060                 if p4PathStartsWith(path, p)]:
2061                 found = False
2062             else:
2063                 found = [p for p in self.depotPaths
2064                          if p4PathStartsWith(path, p)]
2065             if not found:
2066                 fnum = fnum + 1
2067                 continue
2068
2069             file = {}
2070             file["path"] = path
2071             file["rev"] = commit["rev%s" % fnum]
2072             file["action"] = commit["action%s" % fnum]
2073             file["type"] = commit["type%s" % fnum]
2074             files.append(file)
2075             fnum = fnum + 1
2076         return files
2077
2078     def stripRepoPath(self, path, prefixes):
2079         """When streaming files, this is called to map a p4 depot path
2080            to where it should go in git.  The prefixes are either
2081            self.depotPaths, or self.branchPrefixes in the case of
2082            branch detection."""
2083
2084         if self.useClientSpec:
2085             # branch detection moves files up a level (the branch name)
2086             # from what client spec interpretation gives
2087             path = self.clientSpecDirs.map_in_client(path)
2088             if self.detectBranches:
2089                 for b in self.knownBranches:
2090                     if path.startswith(b + "/"):
2091                         path = path[len(b)+1:]
2092
2093         elif self.keepRepoPath:
2094             # Preserve everything in relative path name except leading
2095             # //depot/; just look at first prefix as they all should
2096             # be in the same depot.
2097             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2098             if p4PathStartsWith(path, depot):
2099                 path = path[len(depot):]
2100
2101         else:
2102             for p in prefixes:
2103                 if p4PathStartsWith(path, p):
2104                     path = path[len(p):]
2105                     break
2106
2107         path = wildcard_decode(path)
2108         return path
2109
2110     def splitFilesIntoBranches(self, commit):
2111         """Look at each depotFile in the commit to figure out to what
2112            branch it belongs."""
2113
2114         if self.clientSpecDirs:
2115             files = self.extractFilesFromCommit(commit)
2116             self.clientSpecDirs.update_client_spec_path_cache(files)
2117
2118         branches = {}
2119         fnum = 0
2120         while commit.has_key("depotFile%s" % fnum):
2121             path =  commit["depotFile%s" % fnum]
2122             found = [p for p in self.depotPaths
2123                      if p4PathStartsWith(path, p)]
2124             if not found:
2125                 fnum = fnum + 1
2126                 continue
2127
2128             file = {}
2129             file["path"] = path
2130             file["rev"] = commit["rev%s" % fnum]
2131             file["action"] = commit["action%s" % fnum]
2132             file["type"] = commit["type%s" % fnum]
2133             fnum = fnum + 1
2134
2135             # start with the full relative path where this file would
2136             # go in a p4 client
2137             if self.useClientSpec:
2138                 relPath = self.clientSpecDirs.map_in_client(path)
2139             else:
2140                 relPath = self.stripRepoPath(path, self.depotPaths)
2141
2142             for branch in self.knownBranches.keys():
2143                 # add a trailing slash so that a commit into qt/4.2foo
2144                 # doesn't end up in qt/4.2, e.g.
2145                 if relPath.startswith(branch + "/"):
2146                     if branch not in branches:
2147                         branches[branch] = []
2148                     branches[branch].append(file)
2149                     break
2150
2151         return branches
2152
2153     # output one file from the P4 stream
2154     # - helper for streamP4Files
2155
2156     def streamOneP4File(self, file, contents):
2157         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2158         if verbose:
2159             sys.stderr.write("%s\n" % relPath)
2160
2161         (type_base, type_mods) = split_p4_type(file["type"])
2162
2163         git_mode = "100644"
2164         if "x" in type_mods:
2165             git_mode = "100755"
2166         if type_base == "symlink":
2167             git_mode = "120000"
2168             # p4 print on a symlink sometimes contains "target\n";
2169             # if it does, remove the newline
2170             data = ''.join(contents)
2171             if not data:
2172                 # Some version of p4 allowed creating a symlink that pointed
2173                 # to nothing.  This causes p4 errors when checking out such
2174                 # a change, and errors here too.  Work around it by ignoring
2175                 # the bad symlink; hopefully a future change fixes it.
2176                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2177                 return
2178             elif data[-1] == '\n':
2179                 contents = [data[:-1]]
2180             else:
2181                 contents = [data]
2182
2183         if type_base == "utf16":
2184             # p4 delivers different text in the python output to -G
2185             # than it does when using "print -o", or normal p4 client
2186             # operations.  utf16 is converted to ascii or utf8, perhaps.
2187             # But ascii text saved as -t utf16 is completely mangled.
2188             # Invoke print -o to get the real contents.
2189             #
2190             # On windows, the newlines will always be mangled by print, so put
2191             # them back too.  This is not needed to the cygwin windows version,
2192             # just the native "NT" type.
2193             #
2194             try:
2195                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2196             except Exception as e:
2197                 if 'Translation of file content failed' in str(e):
2198                     type_base = 'binary'
2199                 else:
2200                     raise e
2201             else:
2202                 if p4_version_string().find('/NT') >= 0:
2203                     text = text.replace('\r\n', '\n')
2204                 contents = [ text ]
2205
2206         if type_base == "apple":
2207             # Apple filetype files will be streamed as a concatenation of
2208             # its appledouble header and the contents.  This is useless
2209             # on both macs and non-macs.  If using "print -q -o xx", it
2210             # will create "xx" with the data, and "%xx" with the header.
2211             # This is also not very useful.
2212             #
2213             # Ideally, someday, this script can learn how to generate
2214             # appledouble files directly and import those to git, but
2215             # non-mac machines can never find a use for apple filetype.
2216             print "\nIgnoring apple filetype file %s" % file['depotFile']
2217             return
2218
2219         # Note that we do not try to de-mangle keywords on utf16 files,
2220         # even though in theory somebody may want that.
2221         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2222         if pattern:
2223             regexp = re.compile(pattern, re.VERBOSE)
2224             text = ''.join(contents)
2225             text = regexp.sub(r'$\1$', text)
2226             contents = [ text ]
2227
2228         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2229
2230         # total length...
2231         length = 0
2232         for d in contents:
2233             length = length + len(d)
2234
2235         self.gitStream.write("data %d\n" % length)
2236         for d in contents:
2237             self.gitStream.write(d)
2238         self.gitStream.write("\n")
2239
2240     def streamOneP4Deletion(self, file):
2241         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2242         if verbose:
2243             sys.stderr.write("delete %s\n" % relPath)
2244         self.gitStream.write("D %s\n" % relPath)
2245
2246     # handle another chunk of streaming data
2247     def streamP4FilesCb(self, marshalled):
2248
2249         # catch p4 errors and complain
2250         err = None
2251         if "code" in marshalled:
2252             if marshalled["code"] == "error":
2253                 if "data" in marshalled:
2254                     err = marshalled["data"].rstrip()
2255         if err:
2256             f = None
2257             if self.stream_have_file_info:
2258                 if "depotFile" in self.stream_file:
2259                     f = self.stream_file["depotFile"]
2260             # force a failure in fast-import, else an empty
2261             # commit will be made
2262             self.gitStream.write("\n")
2263             self.gitStream.write("die-now\n")
2264             self.gitStream.close()
2265             # ignore errors, but make sure it exits first
2266             self.importProcess.wait()
2267             if f:
2268                 die("Error from p4 print for %s: %s" % (f, err))
2269             else:
2270                 die("Error from p4 print: %s" % err)
2271
2272         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2273             # start of a new file - output the old one first
2274             self.streamOneP4File(self.stream_file, self.stream_contents)
2275             self.stream_file = {}
2276             self.stream_contents = []
2277             self.stream_have_file_info = False
2278
2279         # pick up the new file information... for the
2280         # 'data' field we need to append to our array
2281         for k in marshalled.keys():
2282             if k == 'data':
2283                 self.stream_contents.append(marshalled['data'])
2284             else:
2285                 self.stream_file[k] = marshalled[k]
2286
2287         self.stream_have_file_info = True
2288
2289     # Stream directly from "p4 files" into "git fast-import"
2290     def streamP4Files(self, files):
2291         filesForCommit = []
2292         filesToRead = []
2293         filesToDelete = []
2294
2295         for f in files:
2296             # if using a client spec, only add the files that have
2297             # a path in the client
2298             if self.clientSpecDirs:
2299                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2300                     continue
2301
2302             filesForCommit.append(f)
2303             if f['action'] in self.delete_actions:
2304                 filesToDelete.append(f)
2305             else:
2306                 filesToRead.append(f)
2307
2308         # deleted files...
2309         for f in filesToDelete:
2310             self.streamOneP4Deletion(f)
2311
2312         if len(filesToRead) > 0:
2313             self.stream_file = {}
2314             self.stream_contents = []
2315             self.stream_have_file_info = False
2316
2317             # curry self argument
2318             def streamP4FilesCbSelf(entry):
2319                 self.streamP4FilesCb(entry)
2320
2321             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2322
2323             p4CmdList(["-x", "-", "print"],
2324                       stdin=fileArgs,
2325                       cb=streamP4FilesCbSelf)
2326
2327             # do the last chunk
2328             if self.stream_file.has_key('depotFile'):
2329                 self.streamOneP4File(self.stream_file, self.stream_contents)
2330
2331     def make_email(self, userid):
2332         if userid in self.users:
2333             return self.users[userid]
2334         else:
2335             return "%s <a@b>" % userid
2336
2337     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2338         """ Stream a p4 tag.
2339         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2340         """
2341
2342         if verbose:
2343             print "writing tag %s for commit %s" % (labelName, commit)
2344         gitStream.write("tag %s\n" % labelName)
2345         gitStream.write("from %s\n" % commit)
2346
2347         if labelDetails.has_key('Owner'):
2348             owner = labelDetails["Owner"]
2349         else:
2350             owner = None
2351
2352         # Try to use the owner of the p4 label, or failing that,
2353         # the current p4 user id.
2354         if owner:
2355             email = self.make_email(owner)
2356         else:
2357             email = self.make_email(self.p4UserId())
2358         tagger = "%s %s %s" % (email, epoch, self.tz)
2359
2360         gitStream.write("tagger %s\n" % tagger)
2361
2362         print "labelDetails=",labelDetails
2363         if labelDetails.has_key('Description'):
2364             description = labelDetails['Description']
2365         else:
2366             description = 'Label from git p4'
2367
2368         gitStream.write("data %d\n" % len(description))
2369         gitStream.write(description)
2370         gitStream.write("\n")
2371
2372     def commit(self, details, files, branch, parent = ""):
2373         epoch = details["time"]
2374         author = details["user"]
2375
2376         if self.verbose:
2377             print "commit into %s" % branch
2378
2379         # start with reading files; if that fails, we should not
2380         # create a commit.
2381         new_files = []
2382         for f in files:
2383             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2384                 new_files.append (f)
2385             else:
2386                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2387
2388         if self.clientSpecDirs:
2389             self.clientSpecDirs.update_client_spec_path_cache(files)
2390
2391         self.gitStream.write("commit %s\n" % branch)
2392         self.gitStream.write("mark :%s\n" % details["change"])
2393         self.committedChanges.add(int(details["change"]))
2394         committer = ""
2395         if author not in self.users:
2396             self.getUserMapFromPerforceServer()
2397         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2398
2399         self.gitStream.write("committer %s\n" % committer)
2400
2401         self.gitStream.write("data <<EOT\n")
2402         self.gitStream.write(details["desc"])
2403         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2404                              (','.join(self.branchPrefixes), details["change"]))
2405         if len(details['options']) > 0:
2406             self.gitStream.write(": options = %s" % details['options'])
2407         self.gitStream.write("]\nEOT\n\n")
2408
2409         if len(parent) > 0:
2410             if self.verbose:
2411                 print "parent %s" % parent
2412             self.gitStream.write("from %s\n" % parent)
2413
2414         self.streamP4Files(new_files)
2415         self.gitStream.write("\n")
2416
2417         change = int(details["change"])
2418
2419         if self.labels.has_key(change):
2420             label = self.labels[change]
2421             labelDetails = label[0]
2422             labelRevisions = label[1]
2423             if self.verbose:
2424                 print "Change %s is labelled %s" % (change, labelDetails)
2425
2426             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2427                                                 for p in self.branchPrefixes])
2428
2429             if len(files) == len(labelRevisions):
2430
2431                 cleanedFiles = {}
2432                 for info in files:
2433                     if info["action"] in self.delete_actions:
2434                         continue
2435                     cleanedFiles[info["depotFile"]] = info["rev"]
2436
2437                 if cleanedFiles == labelRevisions:
2438                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2439
2440                 else:
2441                     if not self.silent:
2442                         print ("Tag %s does not match with change %s: files do not match."
2443                                % (labelDetails["label"], change))
2444
2445             else:
2446                 if not self.silent:
2447                     print ("Tag %s does not match with change %s: file count is different."
2448                            % (labelDetails["label"], change))
2449
2450     # Build a dictionary of changelists and labels, for "detect-labels" option.
2451     def getLabels(self):
2452         self.labels = {}
2453
2454         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2455         if len(l) > 0 and not self.silent:
2456             print "Finding files belonging to labels in %s" % `self.depotPaths`
2457
2458         for output in l:
2459             label = output["label"]
2460             revisions = {}
2461             newestChange = 0
2462             if self.verbose:
2463                 print "Querying files for label %s" % label
2464             for file in p4CmdList(["files"] +
2465                                       ["%s...@%s" % (p, label)
2466                                           for p in self.depotPaths]):
2467                 revisions[file["depotFile"]] = file["rev"]
2468                 change = int(file["change"])
2469                 if change > newestChange:
2470                     newestChange = change
2471
2472             self.labels[newestChange] = [output, revisions]
2473
2474         if self.verbose:
2475             print "Label changes: %s" % self.labels.keys()
2476
2477     # Import p4 labels as git tags. A direct mapping does not
2478     # exist, so assume that if all the files are at the same revision
2479     # then we can use that, or it's something more complicated we should
2480     # just ignore.
2481     def importP4Labels(self, stream, p4Labels):
2482         if verbose:
2483             print "import p4 labels: " + ' '.join(p4Labels)
2484
2485         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2486         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2487         if len(validLabelRegexp) == 0:
2488             validLabelRegexp = defaultLabelRegexp
2489         m = re.compile(validLabelRegexp)
2490
2491         for name in p4Labels:
2492             commitFound = False
2493
2494             if not m.match(name):
2495                 if verbose:
2496                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2497                 continue
2498
2499             if name in ignoredP4Labels:
2500                 continue
2501
2502             labelDetails = p4CmdList(['label', "-o", name])[0]
2503
2504             # get the most recent changelist for each file in this label
2505             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2506                                 for p in self.depotPaths])
2507
2508             if change.has_key('change'):
2509                 # find the corresponding git commit; take the oldest commit
2510                 changelist = int(change['change'])
2511                 if changelist in self.committedChanges:
2512                     gitCommit = ":%d" % changelist       # use a fast-import mark
2513                     commitFound = True
2514                 else:
2515                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2516                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2517                     if len(gitCommit) == 0:
2518                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2519                     else:
2520                         commitFound = True
2521                         gitCommit = gitCommit.strip()
2522
2523                 if commitFound:
2524                     # Convert from p4 time format
2525                     try:
2526                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2527                     except ValueError:
2528                         print "Could not convert label time %s" % labelDetails['Update']
2529                         tmwhen = 1
2530
2531                     when = int(time.mktime(tmwhen))
2532                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2533                     if verbose:
2534                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2535             else:
2536                 if verbose:
2537                     print "Label %s has no changelists - possibly deleted?" % name
2538
2539             if not commitFound:
2540                 # We can't import this label; don't try again as it will get very
2541                 # expensive repeatedly fetching all the files for labels that will
2542                 # never be imported. If the label is moved in the future, the
2543                 # ignore will need to be removed manually.
2544                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2545
2546     def guessProjectName(self):
2547         for p in self.depotPaths:
2548             if p.endswith("/"):
2549                 p = p[:-1]
2550             p = p[p.strip().rfind("/") + 1:]
2551             if not p.endswith("/"):
2552                p += "/"
2553             return p
2554
2555     def getBranchMapping(self):
2556         lostAndFoundBranches = set()
2557
2558         user = gitConfig("git-p4.branchUser")
2559         if len(user) > 0:
2560             command = "branches -u %s" % user
2561         else:
2562             command = "branches"
2563
2564         for info in p4CmdList(command):
2565             details = p4Cmd(["branch", "-o", info["branch"]])
2566             viewIdx = 0
2567             while details.has_key("View%s" % viewIdx):
2568                 paths = details["View%s" % viewIdx].split(" ")
2569                 viewIdx = viewIdx + 1
2570                 # require standard //depot/foo/... //depot/bar/... mapping
2571                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2572                     continue
2573                 source = paths[0]
2574                 destination = paths[1]
2575                 ## HACK
2576                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2577                     source = source[len(self.depotPaths[0]):-4]
2578                     destination = destination[len(self.depotPaths[0]):-4]
2579
2580                     if destination in self.knownBranches:
2581                         if not self.silent:
2582                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2583                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2584                         continue
2585
2586                     self.knownBranches[destination] = source
2587
2588                     lostAndFoundBranches.discard(destination)
2589
2590                     if source not in self.knownBranches:
2591                         lostAndFoundBranches.add(source)
2592
2593         # Perforce does not strictly require branches to be defined, so we also
2594         # check git config for a branch list.
2595         #
2596         # Example of branch definition in git config file:
2597         # [git-p4]
2598         #   branchList=main:branchA
2599         #   branchList=main:branchB
2600         #   branchList=branchA:branchC
2601         configBranches = gitConfigList("git-p4.branchList")
2602         for branch in configBranches:
2603             if branch:
2604                 (source, destination) = branch.split(":")
2605                 self.knownBranches[destination] = source
2606
2607                 lostAndFoundBranches.discard(destination)
2608
2609                 if source not in self.knownBranches:
2610                     lostAndFoundBranches.add(source)
2611
2612
2613         for branch in lostAndFoundBranches:
2614             self.knownBranches[branch] = branch
2615
2616     def getBranchMappingFromGitBranches(self):
2617         branches = p4BranchesInGit(self.importIntoRemotes)
2618         for branch in branches.keys():
2619             if branch == "master":
2620                 branch = "main"
2621             else:
2622                 branch = branch[len(self.projectName):]
2623             self.knownBranches[branch] = branch
2624
2625     def updateOptionDict(self, d):
2626         option_keys = {}
2627         if self.keepRepoPath:
2628             option_keys['keepRepoPath'] = 1
2629
2630         d["options"] = ' '.join(sorted(option_keys.keys()))
2631
2632     def readOptions(self, d):
2633         self.keepRepoPath = (d.has_key('options')
2634                              and ('keepRepoPath' in d['options']))
2635
2636     def gitRefForBranch(self, branch):
2637         if branch == "main":
2638             return self.refPrefix + "master"
2639
2640         if len(branch) <= 0:
2641             return branch
2642
2643         return self.refPrefix + self.projectName + branch
2644
2645     def gitCommitByP4Change(self, ref, change):
2646         if self.verbose:
2647             print "looking in ref " + ref + " for change %s using bisect..." % change
2648
2649         earliestCommit = ""
2650         latestCommit = parseRevision(ref)
2651
2652         while True:
2653             if self.verbose:
2654                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2655             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2656             if len(next) == 0:
2657                 if self.verbose:
2658                     print "argh"
2659                 return ""
2660             log = extractLogMessageFromGitCommit(next)
2661             settings = extractSettingsGitLog(log)
2662             currentChange = int(settings['change'])
2663             if self.verbose:
2664                 print "current change %s" % currentChange
2665
2666             if currentChange == change:
2667                 if self.verbose:
2668                     print "found %s" % next
2669                 return next
2670
2671             if currentChange < change:
2672                 earliestCommit = "^%s" % next
2673             else:
2674                 latestCommit = "%s" % next
2675
2676         return ""
2677
2678     def importNewBranch(self, branch, maxChange):
2679         # make fast-import flush all changes to disk and update the refs using the checkpoint
2680         # command so that we can try to find the branch parent in the git history
2681         self.gitStream.write("checkpoint\n\n");
2682         self.gitStream.flush();
2683         branchPrefix = self.depotPaths[0] + branch + "/"
2684         range = "@1,%s" % maxChange
2685         #print "prefix" + branchPrefix
2686         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2687         if len(changes) <= 0:
2688             return False
2689         firstChange = changes[0]
2690         #print "first change in branch: %s" % firstChange
2691         sourceBranch = self.knownBranches[branch]
2692         sourceDepotPath = self.depotPaths[0] + sourceBranch
2693         sourceRef = self.gitRefForBranch(sourceBranch)
2694         #print "source " + sourceBranch
2695
2696         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2697         #print "branch parent: %s" % branchParentChange
2698         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2699         if len(gitParent) > 0:
2700             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2701             #print "parent git commit: %s" % gitParent
2702
2703         self.importChanges(changes)
2704         return True
2705
2706     def searchParent(self, parent, branch, target):
2707         parentFound = False
2708         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2709                                      "--no-merges", parent]):
2710             blob = blob.strip()
2711             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2712                 parentFound = True
2713                 if self.verbose:
2714                     print "Found parent of %s in commit %s" % (branch, blob)
2715                 break
2716         if parentFound:
2717             return blob
2718         else:
2719             return None
2720
2721     def importChanges(self, changes):
2722         cnt = 1
2723         for change in changes:
2724             description = p4_describe(change)
2725             self.updateOptionDict(description)
2726
2727             if not self.silent:
2728                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2729                 sys.stdout.flush()
2730             cnt = cnt + 1
2731
2732             try:
2733                 if self.detectBranches:
2734                     branches = self.splitFilesIntoBranches(description)
2735                     for branch in branches.keys():
2736                         ## HACK  --hwn
2737                         branchPrefix = self.depotPaths[0] + branch + "/"
2738                         self.branchPrefixes = [ branchPrefix ]
2739
2740                         parent = ""
2741
2742                         filesForCommit = branches[branch]
2743
2744                         if self.verbose:
2745                             print "branch is %s" % branch
2746
2747                         self.updatedBranches.add(branch)
2748
2749                         if branch not in self.createdBranches:
2750                             self.createdBranches.add(branch)
2751                             parent = self.knownBranches[branch]
2752                             if parent == branch:
2753                                 parent = ""
2754                             else:
2755                                 fullBranch = self.projectName + branch
2756                                 if fullBranch not in self.p4BranchesInGit:
2757                                     if not self.silent:
2758                                         print("\n    Importing new branch %s" % fullBranch);
2759                                     if self.importNewBranch(branch, change - 1):
2760                                         parent = ""
2761                                         self.p4BranchesInGit.append(fullBranch)
2762                                     if not self.silent:
2763                                         print("\n    Resuming with change %s" % change);
2764
2765                                 if self.verbose:
2766                                     print "parent determined through known branches: %s" % parent
2767
2768                         branch = self.gitRefForBranch(branch)
2769                         parent = self.gitRefForBranch(parent)
2770
2771                         if self.verbose:
2772                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2773
2774                         if len(parent) == 0 and branch in self.initialParents:
2775                             parent = self.initialParents[branch]
2776                             del self.initialParents[branch]
2777
2778                         blob = None
2779                         if len(parent) > 0:
2780                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2781                             if self.verbose:
2782                                 print "Creating temporary branch: " + tempBranch
2783                             self.commit(description, filesForCommit, tempBranch)
2784                             self.tempBranches.append(tempBranch)
2785                             self.checkpoint()
2786                             blob = self.searchParent(parent, branch, tempBranch)
2787                         if blob:
2788                             self.commit(description, filesForCommit, branch, blob)
2789                         else:
2790                             if self.verbose:
2791                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2792                             self.commit(description, filesForCommit, branch, parent)
2793                 else:
2794                     files = self.extractFilesFromCommit(description)
2795                     self.commit(description, files, self.branch,
2796                                 self.initialParent)
2797                     # only needed once, to connect to the previous commit
2798                     self.initialParent = ""
2799             except IOError:
2800                 print self.gitError.read()
2801                 sys.exit(1)
2802
2803     def importHeadRevision(self, revision):
2804         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2805
2806         details = {}
2807         details["user"] = "git perforce import user"
2808         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2809                            % (' '.join(self.depotPaths), revision))
2810         details["change"] = revision
2811         newestRevision = 0
2812
2813         fileCnt = 0
2814         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2815
2816         for info in p4CmdList(["files"] + fileArgs):
2817
2818             if 'code' in info and info['code'] == 'error':
2819                 sys.stderr.write("p4 returned an error: %s\n"
2820                                  % info['data'])
2821                 if info['data'].find("must refer to client") >= 0:
2822                     sys.stderr.write("This particular p4 error is misleading.\n")
2823                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2824                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2825                 sys.exit(1)
2826             if 'p4ExitCode' in info:
2827                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2828                 sys.exit(1)
2829
2830
2831             change = int(info["change"])
2832             if change > newestRevision:
2833                 newestRevision = change
2834
2835             if info["action"] in self.delete_actions:
2836                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2837                 #fileCnt = fileCnt + 1
2838                 continue
2839
2840             for prop in ["depotFile", "rev", "action", "type" ]:
2841                 details["%s%s" % (prop, fileCnt)] = info[prop]
2842
2843             fileCnt = fileCnt + 1
2844
2845         details["change"] = newestRevision
2846
2847         # Use time from top-most change so that all git p4 clones of
2848         # the same p4 repo have the same commit SHA1s.
2849         res = p4_describe(newestRevision)
2850         details["time"] = res["time"]
2851
2852         self.updateOptionDict(details)
2853         try:
2854             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2855         except IOError:
2856             print "IO error with git fast-import. Is your git version recent enough?"
2857             print self.gitError.read()
2858
2859
2860     def run(self, args):
2861         self.depotPaths = []
2862         self.changeRange = ""
2863         self.previousDepotPaths = []
2864         self.hasOrigin = False
2865
2866         # map from branch depot path to parent branch
2867         self.knownBranches = {}
2868         self.initialParents = {}
2869
2870         if self.importIntoRemotes:
2871             self.refPrefix = "refs/remotes/p4/"
2872         else:
2873             self.refPrefix = "refs/heads/p4/"
2874
2875         if self.syncWithOrigin:
2876             self.hasOrigin = originP4BranchesExist()
2877             if self.hasOrigin:
2878                 if not self.silent:
2879                     print 'Syncing with origin first, using "git fetch origin"'
2880                 system("git fetch origin")
2881
2882         branch_arg_given = bool(self.branch)
2883         if len(self.branch) == 0:
2884             self.branch = self.refPrefix + "master"
2885             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2886                 system("git update-ref %s refs/heads/p4" % self.branch)
2887                 system("git branch -D p4")
2888
2889         # accept either the command-line option, or the configuration variable
2890         if self.useClientSpec:
2891             # will use this after clone to set the variable
2892             self.useClientSpec_from_options = True
2893         else:
2894             if gitConfigBool("git-p4.useclientspec"):
2895                 self.useClientSpec = True
2896         if self.useClientSpec:
2897             self.clientSpecDirs = getClientSpec()
2898
2899         # TODO: should always look at previous commits,
2900         # merge with previous imports, if possible.
2901         if args == []:
2902             if self.hasOrigin:
2903                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2904
2905             # branches holds mapping from branch name to sha1
2906             branches = p4BranchesInGit(self.importIntoRemotes)
2907
2908             # restrict to just this one, disabling detect-branches
2909             if branch_arg_given:
2910                 short = self.branch.split("/")[-1]
2911                 if short in branches:
2912                     self.p4BranchesInGit = [ short ]
2913             else:
2914                 self.p4BranchesInGit = branches.keys()
2915
2916             if len(self.p4BranchesInGit) > 1:
2917                 if not self.silent:
2918                     print "Importing from/into multiple branches"
2919                 self.detectBranches = True
2920                 for branch in branches.keys():
2921                     self.initialParents[self.refPrefix + branch] = \
2922                         branches[branch]
2923
2924             if self.verbose:
2925                 print "branches: %s" % self.p4BranchesInGit
2926
2927             p4Change = 0
2928             for branch in self.p4BranchesInGit:
2929                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2930
2931                 settings = extractSettingsGitLog(logMsg)
2932
2933                 self.readOptions(settings)
2934                 if (settings.has_key('depot-paths')
2935                     and settings.has_key ('change')):
2936                     change = int(settings['change']) + 1
2937                     p4Change = max(p4Change, change)
2938
2939                     depotPaths = sorted(settings['depot-paths'])
2940                     if self.previousDepotPaths == []:
2941                         self.previousDepotPaths = depotPaths
2942                     else:
2943                         paths = []
2944                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2945                             prev_list = prev.split("/")
2946                             cur_list = cur.split("/")
2947                             for i in range(0, min(len(cur_list), len(prev_list))):
2948                                 if cur_list[i] <> prev_list[i]:
2949                                     i = i - 1
2950                                     break
2951
2952                             paths.append ("/".join(cur_list[:i + 1]))
2953
2954                         self.previousDepotPaths = paths
2955
2956             if p4Change > 0:
2957                 self.depotPaths = sorted(self.previousDepotPaths)
2958                 self.changeRange = "@%s,#head" % p4Change
2959                 if not self.silent and not self.detectBranches:
2960                     print "Performing incremental import into %s git branch" % self.branch
2961
2962         # accept multiple ref name abbreviations:
2963         #    refs/foo/bar/branch -> use it exactly
2964         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2965         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2966         if not self.branch.startswith("refs/"):
2967             if self.importIntoRemotes:
2968                 prepend = "refs/remotes/"
2969             else:
2970                 prepend = "refs/heads/"
2971             if not self.branch.startswith("p4/"):
2972                 prepend += "p4/"
2973             self.branch = prepend + self.branch
2974
2975         if len(args) == 0 and self.depotPaths:
2976             if not self.silent:
2977                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2978         else:
2979             if self.depotPaths and self.depotPaths != args:
2980                 print ("previous import used depot path %s and now %s was specified. "
2981                        "This doesn't work!" % (' '.join (self.depotPaths),
2982                                                ' '.join (args)))
2983                 sys.exit(1)
2984
2985             self.depotPaths = sorted(args)
2986
2987         revision = ""
2988         self.users = {}
2989
2990         # Make sure no revision specifiers are used when --changesfile
2991         # is specified.
2992         bad_changesfile = False
2993         if len(self.changesFile) > 0:
2994             for p in self.depotPaths:
2995                 if p.find("@") >= 0 or p.find("#") >= 0:
2996                     bad_changesfile = True
2997                     break
2998         if bad_changesfile:
2999             die("Option --changesfile is incompatible with revision specifiers")
3000
3001         newPaths = []
3002         for p in self.depotPaths:
3003             if p.find("@") != -1:
3004                 atIdx = p.index("@")
3005                 self.changeRange = p[atIdx:]
3006                 if self.changeRange == "@all":
3007                     self.changeRange = ""
3008                 elif ',' not in self.changeRange:
3009                     revision = self.changeRange
3010                     self.changeRange = ""
3011                 p = p[:atIdx]
3012             elif p.find("#") != -1:
3013                 hashIdx = p.index("#")
3014                 revision = p[hashIdx:]
3015                 p = p[:hashIdx]
3016             elif self.previousDepotPaths == []:
3017                 # pay attention to changesfile, if given, else import
3018                 # the entire p4 tree at the head revision
3019                 if len(self.changesFile) == 0:
3020                     revision = "#head"
3021
3022             p = re.sub ("\.\.\.$", "", p)
3023             if not p.endswith("/"):
3024                 p += "/"
3025
3026             newPaths.append(p)
3027
3028         self.depotPaths = newPaths
3029
3030         # --detect-branches may change this for each branch
3031         self.branchPrefixes = self.depotPaths
3032
3033         self.loadUserMapFromCache()
3034         self.labels = {}
3035         if self.detectLabels:
3036             self.getLabels();
3037
3038         if self.detectBranches:
3039             ## FIXME - what's a P4 projectName ?
3040             self.projectName = self.guessProjectName()
3041
3042             if self.hasOrigin:
3043                 self.getBranchMappingFromGitBranches()
3044             else:
3045                 self.getBranchMapping()
3046             if self.verbose:
3047                 print "p4-git branches: %s" % self.p4BranchesInGit
3048                 print "initial parents: %s" % self.initialParents
3049             for b in self.p4BranchesInGit:
3050                 if b != "master":
3051
3052                     ## FIXME
3053                     b = b[len(self.projectName):]
3054                 self.createdBranches.add(b)
3055
3056         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3057
3058         self.importProcess = subprocess.Popen(["git", "fast-import"],
3059                                               stdin=subprocess.PIPE,
3060                                               stdout=subprocess.PIPE,
3061                                               stderr=subprocess.PIPE);
3062         self.gitOutput = self.importProcess.stdout
3063         self.gitStream = self.importProcess.stdin
3064         self.gitError = self.importProcess.stderr
3065
3066         if revision:
3067             self.importHeadRevision(revision)
3068         else:
3069             changes = []
3070
3071             if len(self.changesFile) > 0:
3072                 output = open(self.changesFile).readlines()
3073                 changeSet = set()
3074                 for line in output:
3075                     changeSet.add(int(line))
3076
3077                 for change in changeSet:
3078                     changes.append(change)
3079
3080                 changes.sort()
3081             else:
3082                 # catch "git p4 sync" with no new branches, in a repo that
3083                 # does not have any existing p4 branches
3084                 if len(args) == 0:
3085                     if not self.p4BranchesInGit:
3086                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3087
3088                     # The default branch is master, unless --branch is used to
3089                     # specify something else.  Make sure it exists, or complain
3090                     # nicely about how to use --branch.
3091                     if not self.detectBranches:
3092                         if not branch_exists(self.branch):
3093                             if branch_arg_given:
3094                                 die("Error: branch %s does not exist." % self.branch)
3095                             else:
3096                                 die("Error: no branch %s; perhaps specify one with --branch." %
3097                                     self.branch)
3098
3099                 if self.verbose:
3100                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3101                                                               self.changeRange)
3102                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3103
3104                 if len(self.maxChanges) > 0:
3105                     changes = changes[:min(int(self.maxChanges), len(changes))]
3106
3107             if len(changes) == 0:
3108                 if not self.silent:
3109                     print "No changes to import!"
3110             else:
3111                 if not self.silent and not self.detectBranches:
3112                     print "Import destination: %s" % self.branch
3113
3114                 self.updatedBranches = set()
3115
3116                 if not self.detectBranches:
3117                     if args:
3118                         # start a new branch
3119                         self.initialParent = ""
3120                     else:
3121                         # build on a previous revision
3122                         self.initialParent = parseRevision(self.branch)
3123
3124                 self.importChanges(changes)
3125
3126                 if not self.silent:
3127                     print ""
3128                     if len(self.updatedBranches) > 0:
3129                         sys.stdout.write("Updated branches: ")
3130                         for b in self.updatedBranches:
3131                             sys.stdout.write("%s " % b)
3132                         sys.stdout.write("\n")
3133
3134         if gitConfigBool("git-p4.importLabels"):
3135             self.importLabels = True
3136
3137         if self.importLabels:
3138             p4Labels = getP4Labels(self.depotPaths)
3139             gitTags = getGitTags()
3140
3141             missingP4Labels = p4Labels - gitTags
3142             self.importP4Labels(self.gitStream, missingP4Labels)
3143
3144         self.gitStream.close()
3145         if self.importProcess.wait() != 0:
3146             die("fast-import failed: %s" % self.gitError.read())
3147         self.gitOutput.close()
3148         self.gitError.close()
3149
3150         # Cleanup temporary branches created during import
3151         if self.tempBranches != []:
3152             for branch in self.tempBranches:
3153                 read_pipe("git update-ref -d %s" % branch)
3154             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3155
3156         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3157         # a convenient shortcut refname "p4".
3158         if self.importIntoRemotes:
3159             head_ref = self.refPrefix + "HEAD"
3160             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3161                 system(["git", "symbolic-ref", head_ref, self.branch])
3162
3163         return True
3164
3165 class P4Rebase(Command):
3166     def __init__(self):
3167         Command.__init__(self)
3168         self.options = [
3169                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3170         ]
3171         self.importLabels = False
3172         self.description = ("Fetches the latest revision from perforce and "
3173                             + "rebases the current work (branch) against it")
3174
3175     def run(self, args):
3176         sync = P4Sync()
3177         sync.importLabels = self.importLabels
3178         sync.run([])
3179
3180         return self.rebase()
3181
3182     def rebase(self):
3183         if os.system("git update-index --refresh") != 0:
3184             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3185         if len(read_pipe("git diff-index HEAD --")) > 0:
3186             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3187
3188         [upstream, settings] = findUpstreamBranchPoint()
3189         if len(upstream) == 0:
3190             die("Cannot find upstream branchpoint for rebase")
3191
3192         # the branchpoint may be p4/foo~3, so strip off the parent
3193         upstream = re.sub("~[0-9]+$", "", upstream)
3194
3195         print "Rebasing the current branch onto %s" % upstream
3196         oldHead = read_pipe("git rev-parse HEAD").strip()
3197         system("git rebase %s" % upstream)
3198         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3199         return True
3200
3201 class P4Clone(P4Sync):
3202     def __init__(self):
3203         P4Sync.__init__(self)
3204         self.description = "Creates a new git repository and imports from Perforce into it"
3205         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3206         self.options += [
3207             optparse.make_option("--destination", dest="cloneDestination",
3208                                  action='store', default=None,
3209                                  help="where to leave result of the clone"),
3210             optparse.make_option("--bare", dest="cloneBare",
3211                                  action="store_true", default=False),
3212         ]
3213         self.cloneDestination = None
3214         self.needsGit = False
3215         self.cloneBare = False
3216
3217     def defaultDestination(self, args):
3218         ## TODO: use common prefix of args?
3219         depotPath = args[0]
3220         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3221         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3222         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3223         depotDir = re.sub(r"/$", "", depotDir)
3224         return os.path.split(depotDir)[1]
3225
3226     def run(self, args):
3227         if len(args) < 1:
3228             return False
3229
3230         if self.keepRepoPath and not self.cloneDestination:
3231             sys.stderr.write("Must specify destination for --keep-path\n")
3232             sys.exit(1)
3233
3234         depotPaths = args
3235
3236         if not self.cloneDestination and len(depotPaths) > 1:
3237             self.cloneDestination = depotPaths[-1]
3238             depotPaths = depotPaths[:-1]
3239
3240         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3241         for p in depotPaths:
3242             if not p.startswith("//"):
3243                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3244                 return False
3245
3246         if not self.cloneDestination:
3247             self.cloneDestination = self.defaultDestination(args)
3248
3249         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3250
3251         if not os.path.exists(self.cloneDestination):
3252             os.makedirs(self.cloneDestination)
3253         chdir(self.cloneDestination)
3254
3255         init_cmd = [ "git", "init" ]
3256         if self.cloneBare:
3257             init_cmd.append("--bare")
3258         retcode = subprocess.call(init_cmd)
3259         if retcode:
3260             raise CalledProcessError(retcode, init_cmd)
3261
3262         if not P4Sync.run(self, depotPaths):
3263             return False
3264
3265         # create a master branch and check out a work tree
3266         if gitBranchExists(self.branch):
3267             system([ "git", "branch", "master", self.branch ])
3268             if not self.cloneBare:
3269                 system([ "git", "checkout", "-f" ])
3270         else:
3271             print 'Not checking out any branch, use ' \
3272                   '"git checkout -q -b master <branch>"'
3273
3274         # auto-set this variable if invoked with --use-client-spec
3275         if self.useClientSpec_from_options:
3276             system("git config --bool git-p4.useclientspec true")
3277
3278         return True
3279
3280 class P4Branches(Command):
3281     def __init__(self):
3282         Command.__init__(self)
3283         self.options = [ ]
3284         self.description = ("Shows the git branches that hold imports and their "
3285                             + "corresponding perforce depot paths")
3286         self.verbose = False
3287
3288     def run(self, args):
3289         if originP4BranchesExist():
3290             createOrUpdateBranchesFromOrigin()
3291
3292         cmdline = "git rev-parse --symbolic "
3293         cmdline += " --remotes"
3294
3295         for line in read_pipe_lines(cmdline):
3296             line = line.strip()
3297
3298             if not line.startswith('p4/') or line == "p4/HEAD":
3299                 continue
3300             branch = line
3301
3302             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3303             settings = extractSettingsGitLog(log)
3304
3305             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3306         return True
3307
3308 class HelpFormatter(optparse.IndentedHelpFormatter):
3309     def __init__(self):
3310         optparse.IndentedHelpFormatter.__init__(self)
3311
3312     def format_description(self, description):
3313         if description:
3314             return description + "\n"
3315         else:
3316             return ""
3317
3318 def printUsage(commands):
3319     print "usage: %s <command> [options]" % sys.argv[0]
3320     print ""
3321     print "valid commands: %s" % ", ".join(commands)
3322     print ""
3323     print "Try %s <command> --help for command specific help." % sys.argv[0]
3324     print ""
3325
3326 commands = {
3327     "debug" : P4Debug,
3328     "submit" : P4Submit,
3329     "commit" : P4Submit,
3330     "sync" : P4Sync,
3331     "rebase" : P4Rebase,
3332     "clone" : P4Clone,
3333     "rollback" : P4RollBack,
3334     "branches" : P4Branches
3335 }
3336
3337
3338 def main():
3339     if len(sys.argv[1:]) == 0:
3340         printUsage(commands.keys())
3341         sys.exit(2)
3342
3343     cmdName = sys.argv[1]
3344     try:
3345         klass = commands[cmdName]
3346         cmd = klass()
3347     except KeyError:
3348         print "unknown command %s" % cmdName
3349         print ""
3350         printUsage(commands.keys())
3351         sys.exit(2)
3352
3353     options = cmd.options
3354     cmd.gitdir = os.environ.get("GIT_DIR", None)
3355
3356     args = sys.argv[2:]
3357
3358     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3359     if cmd.needsGit:
3360         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3361
3362     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3363                                    options,
3364                                    description = cmd.description,
3365                                    formatter = HelpFormatter())
3366
3367     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3368     global verbose
3369     verbose = cmd.verbose
3370     if cmd.needsGit:
3371         if cmd.gitdir == None:
3372             cmd.gitdir = os.path.abspath(".git")
3373             if not isValidGitDir(cmd.gitdir):
3374                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3375                 if os.path.exists(cmd.gitdir):
3376                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3377                     if len(cdup) > 0:
3378                         chdir(cdup);
3379
3380         if not isValidGitDir(cmd.gitdir):
3381             if isValidGitDir(cmd.gitdir + "/.git"):
3382                 cmd.gitdir += "/.git"
3383             else:
3384                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3385
3386         os.environ["GIT_DIR"] = cmd.gitdir
3387
3388     if not cmd.run(args):
3389         parser.print_help()
3390         sys.exit(2)
3391
3392
3393 if __name__ == '__main__':
3394     main()