Merge branch 'jk/diff-algo-finishing-touches' into maint
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27     from subprocess import CalledProcessError
28 except ImportError:
29     # from python2.7:subprocess.py
30     # Exception classes used by this module.
31     class CalledProcessError(Exception):
32         """This exception is raised when a process run by check_call() returns
33         a non-zero exit status.  The exit status will be stored in the
34         returncode attribute."""
35         def __init__(self, returncode, cmd):
36             self.returncode = returncode
37             self.cmd = cmd
38         def __str__(self):
39             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 def p4_build_cmd(cmd):
47     """Build a suitable p4 command line.
48
49     This consolidates building and returning a p4 command line into one
50     location. It means that hooking into the environment, or other configuration
51     can be done more easily.
52     """
53     real_cmd = ["p4"]
54
55     user = gitConfig("git-p4.user")
56     if len(user) > 0:
57         real_cmd += ["-u",user]
58
59     password = gitConfig("git-p4.password")
60     if len(password) > 0:
61         real_cmd += ["-P", password]
62
63     port = gitConfig("git-p4.port")
64     if len(port) > 0:
65         real_cmd += ["-p", port]
66
67     host = gitConfig("git-p4.host")
68     if len(host) > 0:
69         real_cmd += ["-H", host]
70
71     client = gitConfig("git-p4.client")
72     if len(client) > 0:
73         real_cmd += ["-c", client]
74
75
76     if isinstance(cmd,basestring):
77         real_cmd = ' '.join(real_cmd) + ' ' + cmd
78     else:
79         real_cmd += cmd
80     return real_cmd
81
82 def chdir(dir):
83     # P4 uses the PWD environment variable rather than getcwd(). Since we're
84     # not using the shell, we have to set it ourselves.  This path could
85     # be relative, so go there first, then figure out where we ended up.
86     os.chdir(dir)
87     os.environ['PWD'] = os.getcwd()
88
89 def die(msg):
90     if verbose:
91         raise Exception(msg)
92     else:
93         sys.stderr.write(msg + "\n")
94         sys.exit(1)
95
96 def write_pipe(c, stdin):
97     if verbose:
98         sys.stderr.write('Writing pipe: %s\n' % str(c))
99
100     expand = isinstance(c,basestring)
101     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
102     pipe = p.stdin
103     val = pipe.write(stdin)
104     pipe.close()
105     if p.wait():
106         die('Command failed: %s' % str(c))
107
108     return val
109
110 def p4_write_pipe(c, stdin):
111     real_cmd = p4_build_cmd(c)
112     return write_pipe(real_cmd, stdin)
113
114 def read_pipe(c, ignore_error=False):
115     if verbose:
116         sys.stderr.write('Reading pipe: %s\n' % str(c))
117
118     expand = isinstance(c,basestring)
119     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
120     pipe = p.stdout
121     val = pipe.read()
122     if p.wait() and not ignore_error:
123         die('Command failed: %s' % str(c))
124
125     return val
126
127 def p4_read_pipe(c, ignore_error=False):
128     real_cmd = p4_build_cmd(c)
129     return read_pipe(real_cmd, ignore_error)
130
131 def read_pipe_lines(c):
132     if verbose:
133         sys.stderr.write('Reading pipe: %s\n' % str(c))
134
135     expand = isinstance(c, basestring)
136     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
137     pipe = p.stdout
138     val = pipe.readlines()
139     if pipe.close() or p.wait():
140         die('Command failed: %s' % str(c))
141
142     return val
143
144 def p4_read_pipe_lines(c):
145     """Specifically invoke p4 on the command supplied. """
146     real_cmd = p4_build_cmd(c)
147     return read_pipe_lines(real_cmd)
148
149 def p4_has_command(cmd):
150     """Ask p4 for help on this command.  If it returns an error, the
151        command does not exist in this version of p4."""
152     real_cmd = p4_build_cmd(["help", cmd])
153     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
154                                    stderr=subprocess.PIPE)
155     p.communicate()
156     return p.returncode == 0
157
158 def p4_has_move_command():
159     """See if the move command exists, that it supports -k, and that
160        it has not been administratively disabled.  The arguments
161        must be correct, but the filenames do not have to exist.  Use
162        ones with wildcards so even if they exist, it will fail."""
163
164     if not p4_has_command("move"):
165         return False
166     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
167     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
168     (out, err) = p.communicate()
169     # return code will be 1 in either case
170     if err.find("Invalid option") >= 0:
171         return False
172     if err.find("disabled") >= 0:
173         return False
174     # assume it failed because @... was invalid changelist
175     return True
176
177 def system(cmd):
178     expand = isinstance(cmd,basestring)
179     if verbose:
180         sys.stderr.write("executing %s\n" % str(cmd))
181     retcode = subprocess.call(cmd, shell=expand)
182     if retcode:
183         raise CalledProcessError(retcode, cmd)
184
185 def p4_system(cmd):
186     """Specifically invoke p4 as the system command. """
187     real_cmd = p4_build_cmd(cmd)
188     expand = isinstance(real_cmd, basestring)
189     retcode = subprocess.call(real_cmd, shell=expand)
190     if retcode:
191         raise CalledProcessError(retcode, real_cmd)
192
193 _p4_version_string = None
194 def p4_version_string():
195     """Read the version string, showing just the last line, which
196        hopefully is the interesting version bit.
197
198        $ p4 -V
199        Perforce - The Fast Software Configuration Management System.
200        Copyright 1995-2011 Perforce Software.  All rights reserved.
201        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
202     """
203     global _p4_version_string
204     if not _p4_version_string:
205         a = p4_read_pipe_lines(["-V"])
206         _p4_version_string = a[-1].rstrip()
207     return _p4_version_string
208
209 def p4_integrate(src, dest):
210     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
211
212 def p4_sync(f, *options):
213     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
214
215 def p4_add(f):
216     # forcibly add file names with wildcards
217     if wildcard_present(f):
218         p4_system(["add", "-f", f])
219     else:
220         p4_system(["add", f])
221
222 def p4_delete(f):
223     p4_system(["delete", wildcard_encode(f)])
224
225 def p4_edit(f):
226     p4_system(["edit", wildcard_encode(f)])
227
228 def p4_revert(f):
229     p4_system(["revert", wildcard_encode(f)])
230
231 def p4_reopen(type, f):
232     p4_system(["reopen", "-t", type, wildcard_encode(f)])
233
234 def p4_move(src, dest):
235     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
236
237 def p4_describe(change):
238     """Make sure it returns a valid result by checking for
239        the presence of field "time".  Return a dict of the
240        results."""
241
242     ds = p4CmdList(["describe", "-s", str(change)])
243     if len(ds) != 1:
244         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
245
246     d = ds[0]
247
248     if "p4ExitCode" in d:
249         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
250                                                       str(d)))
251     if "code" in d:
252         if d["code"] == "error":
253             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
254
255     if "time" not in d:
256         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
257
258     return d
259
260 #
261 # Canonicalize the p4 type and return a tuple of the
262 # base type, plus any modifiers.  See "p4 help filetypes"
263 # for a list and explanation.
264 #
265 def split_p4_type(p4type):
266
267     p4_filetypes_historical = {
268         "ctempobj": "binary+Sw",
269         "ctext": "text+C",
270         "cxtext": "text+Cx",
271         "ktext": "text+k",
272         "kxtext": "text+kx",
273         "ltext": "text+F",
274         "tempobj": "binary+FSw",
275         "ubinary": "binary+F",
276         "uresource": "resource+F",
277         "uxbinary": "binary+Fx",
278         "xbinary": "binary+x",
279         "xltext": "text+Fx",
280         "xtempobj": "binary+Swx",
281         "xtext": "text+x",
282         "xunicode": "unicode+x",
283         "xutf16": "utf16+x",
284     }
285     if p4type in p4_filetypes_historical:
286         p4type = p4_filetypes_historical[p4type]
287     mods = ""
288     s = p4type.split("+")
289     base = s[0]
290     mods = ""
291     if len(s) > 1:
292         mods = s[1]
293     return (base, mods)
294
295 #
296 # return the raw p4 type of a file (text, text+ko, etc)
297 #
298 def p4_type(file):
299     results = p4CmdList(["fstat", "-T", "headType", file])
300     return results[0]['headType']
301
302 #
303 # Given a type base and modifier, return a regexp matching
304 # the keywords that can be expanded in the file
305 #
306 def p4_keywords_regexp_for_type(base, type_mods):
307     if base in ("text", "unicode", "binary"):
308         kwords = None
309         if "ko" in type_mods:
310             kwords = 'Id|Header'
311         elif "k" in type_mods:
312             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
313         else:
314             return None
315         pattern = r"""
316             \$              # Starts with a dollar, followed by...
317             (%s)            # one of the keywords, followed by...
318             (:[^$\n]+)?     # possibly an old expansion, followed by...
319             \$              # another dollar
320             """ % kwords
321         return pattern
322     else:
323         return None
324
325 #
326 # Given a file, return a regexp matching the possible
327 # RCS keywords that will be expanded, or None for files
328 # with kw expansion turned off.
329 #
330 def p4_keywords_regexp_for_file(file):
331     if not os.path.exists(file):
332         return None
333     else:
334         (type_base, type_mods) = split_p4_type(p4_type(file))
335         return p4_keywords_regexp_for_type(type_base, type_mods)
336
337 def setP4ExecBit(file, mode):
338     # Reopens an already open file and changes the execute bit to match
339     # the execute bit setting in the passed in mode.
340
341     p4Type = "+x"
342
343     if not isModeExec(mode):
344         p4Type = getP4OpenedType(file)
345         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
346         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
347         if p4Type[-1] == "+":
348             p4Type = p4Type[0:-1]
349
350     p4_reopen(p4Type, file)
351
352 def getP4OpenedType(file):
353     # Returns the perforce file type for the given file.
354
355     result = p4_read_pipe(["opened", wildcard_encode(file)])
356     match = re.match(".*\((.+)\)\r?$", result)
357     if match:
358         return match.group(1)
359     else:
360         die("Could not determine file type for %s (result: '%s')" % (file, result))
361
362 # Return the set of all p4 labels
363 def getP4Labels(depotPaths):
364     labels = set()
365     if isinstance(depotPaths,basestring):
366         depotPaths = [depotPaths]
367
368     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
369         label = l['label']
370         labels.add(label)
371
372     return labels
373
374 # Return the set of all git tags
375 def getGitTags():
376     gitTags = set()
377     for line in read_pipe_lines(["git", "tag"]):
378         tag = line.strip()
379         gitTags.add(tag)
380     return gitTags
381
382 def diffTreePattern():
383     # This is a simple generator for the diff tree regex pattern. This could be
384     # a class variable if this and parseDiffTreeEntry were a part of a class.
385     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
386     while True:
387         yield pattern
388
389 def parseDiffTreeEntry(entry):
390     """Parses a single diff tree entry into its component elements.
391
392     See git-diff-tree(1) manpage for details about the format of the diff
393     output. This method returns a dictionary with the following elements:
394
395     src_mode - The mode of the source file
396     dst_mode - The mode of the destination file
397     src_sha1 - The sha1 for the source file
398     dst_sha1 - The sha1 fr the destination file
399     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
400     status_score - The score for the status (applicable for 'C' and 'R'
401                    statuses). This is None if there is no score.
402     src - The path for the source file.
403     dst - The path for the destination file. This is only present for
404           copy or renames. If it is not present, this is None.
405
406     If the pattern is not matched, None is returned."""
407
408     match = diffTreePattern().next().match(entry)
409     if match:
410         return {
411             'src_mode': match.group(1),
412             'dst_mode': match.group(2),
413             'src_sha1': match.group(3),
414             'dst_sha1': match.group(4),
415             'status': match.group(5),
416             'status_score': match.group(6),
417             'src': match.group(7),
418             'dst': match.group(10)
419         }
420     return None
421
422 def isModeExec(mode):
423     # Returns True if the given git mode represents an executable file,
424     # otherwise False.
425     return mode[-3:] == "755"
426
427 def isModeExecChanged(src_mode, dst_mode):
428     return isModeExec(src_mode) != isModeExec(dst_mode)
429
430 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
431
432     if isinstance(cmd,basestring):
433         cmd = "-G " + cmd
434         expand = True
435     else:
436         cmd = ["-G"] + cmd
437         expand = False
438
439     cmd = p4_build_cmd(cmd)
440     if verbose:
441         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
442
443     # Use a temporary file to avoid deadlocks without
444     # subprocess.communicate(), which would put another copy
445     # of stdout into memory.
446     stdin_file = None
447     if stdin is not None:
448         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
449         if isinstance(stdin,basestring):
450             stdin_file.write(stdin)
451         else:
452             for i in stdin:
453                 stdin_file.write(i + '\n')
454         stdin_file.flush()
455         stdin_file.seek(0)
456
457     p4 = subprocess.Popen(cmd,
458                           shell=expand,
459                           stdin=stdin_file,
460                           stdout=subprocess.PIPE)
461
462     result = []
463     try:
464         while True:
465             entry = marshal.load(p4.stdout)
466             if cb is not None:
467                 cb(entry)
468             else:
469                 result.append(entry)
470     except EOFError:
471         pass
472     exitCode = p4.wait()
473     if exitCode != 0:
474         entry = {}
475         entry["p4ExitCode"] = exitCode
476         result.append(entry)
477
478     return result
479
480 def p4Cmd(cmd):
481     list = p4CmdList(cmd)
482     result = {}
483     for entry in list:
484         result.update(entry)
485     return result;
486
487 def p4Where(depotPath):
488     if not depotPath.endswith("/"):
489         depotPath += "/"
490     depotPath = depotPath + "..."
491     outputList = p4CmdList(["where", depotPath])
492     output = None
493     for entry in outputList:
494         if "depotFile" in entry:
495             if entry["depotFile"] == depotPath:
496                 output = entry
497                 break
498         elif "data" in entry:
499             data = entry.get("data")
500             space = data.find(" ")
501             if data[:space] == depotPath:
502                 output = entry
503                 break
504     if output == None:
505         return ""
506     if output["code"] == "error":
507         return ""
508     clientPath = ""
509     if "path" in output:
510         clientPath = output.get("path")
511     elif "data" in output:
512         data = output.get("data")
513         lastSpace = data.rfind(" ")
514         clientPath = data[lastSpace + 1:]
515
516     if clientPath.endswith("..."):
517         clientPath = clientPath[:-3]
518     return clientPath
519
520 def currentGitBranch():
521     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
522
523 def isValidGitDir(path):
524     if (os.path.exists(path + "/HEAD")
525         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
526         return True;
527     return False
528
529 def parseRevision(ref):
530     return read_pipe("git rev-parse %s" % ref).strip()
531
532 def branchExists(ref):
533     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
534                      ignore_error=True)
535     return len(rev) > 0
536
537 def extractLogMessageFromGitCommit(commit):
538     logMessage = ""
539
540     ## fixme: title is first line of commit, not 1st paragraph.
541     foundTitle = False
542     for log in read_pipe_lines("git cat-file commit %s" % commit):
543        if not foundTitle:
544            if len(log) == 1:
545                foundTitle = True
546            continue
547
548        logMessage += log
549     return logMessage
550
551 def extractSettingsGitLog(log):
552     values = {}
553     for line in log.split("\n"):
554         line = line.strip()
555         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
556         if not m:
557             continue
558
559         assignments = m.group(1).split (':')
560         for a in assignments:
561             vals = a.split ('=')
562             key = vals[0].strip()
563             val = ('='.join (vals[1:])).strip()
564             if val.endswith ('\"') and val.startswith('"'):
565                 val = val[1:-1]
566
567             values[key] = val
568
569     paths = values.get("depot-paths")
570     if not paths:
571         paths = values.get("depot-path")
572     if paths:
573         values['depot-paths'] = paths.split(',')
574     return values
575
576 def gitBranchExists(branch):
577     proc = subprocess.Popen(["git", "rev-parse", branch],
578                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
579     return proc.wait() == 0;
580
581 _gitConfig = {}
582
583 def gitConfig(key):
584     if not _gitConfig.has_key(key):
585         cmd = [ "git", "config", key ]
586         s = read_pipe(cmd, ignore_error=True)
587         _gitConfig[key] = s.strip()
588     return _gitConfig[key]
589
590 def gitConfigBool(key):
591     """Return a bool, using git config --bool.  It is True only if the
592        variable is set to true, and False if set to false or not present
593        in the config."""
594
595     if not _gitConfig.has_key(key):
596         cmd = [ "git", "config", "--bool", key ]
597         s = read_pipe(cmd, ignore_error=True)
598         v = s.strip()
599         _gitConfig[key] = v == "true"
600     return _gitConfig[key]
601
602 def gitConfigList(key):
603     if not _gitConfig.has_key(key):
604         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
605         _gitConfig[key] = s.strip().split(os.linesep)
606     return _gitConfig[key]
607
608 def p4BranchesInGit(branchesAreInRemotes=True):
609     """Find all the branches whose names start with "p4/", looking
610        in remotes or heads as specified by the argument.  Return
611        a dictionary of { branch: revision } for each one found.
612        The branch names are the short names, without any
613        "p4/" prefix."""
614
615     branches = {}
616
617     cmdline = "git rev-parse --symbolic "
618     if branchesAreInRemotes:
619         cmdline += "--remotes"
620     else:
621         cmdline += "--branches"
622
623     for line in read_pipe_lines(cmdline):
624         line = line.strip()
625
626         # only import to p4/
627         if not line.startswith('p4/'):
628             continue
629         # special symbolic ref to p4/master
630         if line == "p4/HEAD":
631             continue
632
633         # strip off p4/ prefix
634         branch = line[len("p4/"):]
635
636         branches[branch] = parseRevision(line)
637
638     return branches
639
640 def branch_exists(branch):
641     """Make sure that the given ref name really exists."""
642
643     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
644     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
645     out, _ = p.communicate()
646     if p.returncode:
647         return False
648     # expect exactly one line of output: the branch name
649     return out.rstrip() == branch
650
651 def findUpstreamBranchPoint(head = "HEAD"):
652     branches = p4BranchesInGit()
653     # map from depot-path to branch name
654     branchByDepotPath = {}
655     for branch in branches.keys():
656         tip = branches[branch]
657         log = extractLogMessageFromGitCommit(tip)
658         settings = extractSettingsGitLog(log)
659         if settings.has_key("depot-paths"):
660             paths = ",".join(settings["depot-paths"])
661             branchByDepotPath[paths] = "remotes/p4/" + branch
662
663     settings = None
664     parent = 0
665     while parent < 65535:
666         commit = head + "~%s" % parent
667         log = extractLogMessageFromGitCommit(commit)
668         settings = extractSettingsGitLog(log)
669         if settings.has_key("depot-paths"):
670             paths = ",".join(settings["depot-paths"])
671             if branchByDepotPath.has_key(paths):
672                 return [branchByDepotPath[paths], settings]
673
674         parent = parent + 1
675
676     return ["", settings]
677
678 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
679     if not silent:
680         print ("Creating/updating branch(es) in %s based on origin branch(es)"
681                % localRefPrefix)
682
683     originPrefix = "origin/p4/"
684
685     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
686         line = line.strip()
687         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
688             continue
689
690         headName = line[len(originPrefix):]
691         remoteHead = localRefPrefix + headName
692         originHead = line
693
694         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
695         if (not original.has_key('depot-paths')
696             or not original.has_key('change')):
697             continue
698
699         update = False
700         if not gitBranchExists(remoteHead):
701             if verbose:
702                 print "creating %s" % remoteHead
703             update = True
704         else:
705             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
706             if settings.has_key('change') > 0:
707                 if settings['depot-paths'] == original['depot-paths']:
708                     originP4Change = int(original['change'])
709                     p4Change = int(settings['change'])
710                     if originP4Change > p4Change:
711                         print ("%s (%s) is newer than %s (%s). "
712                                "Updating p4 branch from origin."
713                                % (originHead, originP4Change,
714                                   remoteHead, p4Change))
715                         update = True
716                 else:
717                     print ("Ignoring: %s was imported from %s while "
718                            "%s was imported from %s"
719                            % (originHead, ','.join(original['depot-paths']),
720                               remoteHead, ','.join(settings['depot-paths'])))
721
722         if update:
723             system("git update-ref %s %s" % (remoteHead, originHead))
724
725 def originP4BranchesExist():
726         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
727
728 def p4ChangesForPaths(depotPaths, changeRange):
729     assert depotPaths
730     cmd = ['changes']
731     for p in depotPaths:
732         cmd += ["%s...%s" % (p, changeRange)]
733     output = p4_read_pipe_lines(cmd)
734
735     changes = {}
736     for line in output:
737         changeNum = int(line.split(" ")[1])
738         changes[changeNum] = True
739
740     changelist = changes.keys()
741     changelist.sort()
742     return changelist
743
744 def p4PathStartsWith(path, prefix):
745     # This method tries to remedy a potential mixed-case issue:
746     #
747     # If UserA adds  //depot/DirA/file1
748     # and UserB adds //depot/dira/file2
749     #
750     # we may or may not have a problem. If you have core.ignorecase=true,
751     # we treat DirA and dira as the same directory
752     if gitConfigBool("core.ignorecase"):
753         return path.lower().startswith(prefix.lower())
754     return path.startswith(prefix)
755
756 def getClientSpec():
757     """Look at the p4 client spec, create a View() object that contains
758        all the mappings, and return it."""
759
760     specList = p4CmdList("client -o")
761     if len(specList) != 1:
762         die('Output from "client -o" is %d lines, expecting 1' %
763             len(specList))
764
765     # dictionary of all client parameters
766     entry = specList[0]
767
768     # just the keys that start with "View"
769     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
770
771     # hold this new View
772     view = View()
773
774     # append the lines, in order, to the view
775     for view_num in range(len(view_keys)):
776         k = "View%d" % view_num
777         if k not in view_keys:
778             die("Expected view key %s missing" % k)
779         view.append(entry[k])
780
781     return view
782
783 def getClientRoot():
784     """Grab the client directory."""
785
786     output = p4CmdList("client -o")
787     if len(output) != 1:
788         die('Output from "client -o" is %d lines, expecting 1' % len(output))
789
790     entry = output[0]
791     if "Root" not in entry:
792         die('Client has no "Root"')
793
794     return entry["Root"]
795
796 #
797 # P4 wildcards are not allowed in filenames.  P4 complains
798 # if you simply add them, but you can force it with "-f", in
799 # which case it translates them into %xx encoding internally.
800 #
801 def wildcard_decode(path):
802     # Search for and fix just these four characters.  Do % last so
803     # that fixing it does not inadvertently create new %-escapes.
804     # Cannot have * in a filename in windows; untested as to
805     # what p4 would do in such a case.
806     if not platform.system() == "Windows":
807         path = path.replace("%2A", "*")
808     path = path.replace("%23", "#") \
809                .replace("%40", "@") \
810                .replace("%25", "%")
811     return path
812
813 def wildcard_encode(path):
814     # do % first to avoid double-encoding the %s introduced here
815     path = path.replace("%", "%25") \
816                .replace("*", "%2A") \
817                .replace("#", "%23") \
818                .replace("@", "%40")
819     return path
820
821 def wildcard_present(path):
822     m = re.search("[*#@%]", path)
823     return m is not None
824
825 class Command:
826     def __init__(self):
827         self.usage = "usage: %prog [options]"
828         self.needsGit = True
829         self.verbose = False
830
831 class P4UserMap:
832     def __init__(self):
833         self.userMapFromPerforceServer = False
834         self.myP4UserId = None
835
836     def p4UserId(self):
837         if self.myP4UserId:
838             return self.myP4UserId
839
840         results = p4CmdList("user -o")
841         for r in results:
842             if r.has_key('User'):
843                 self.myP4UserId = r['User']
844                 return r['User']
845         die("Could not find your p4 user id")
846
847     def p4UserIsMe(self, p4User):
848         # return True if the given p4 user is actually me
849         me = self.p4UserId()
850         if not p4User or p4User != me:
851             return False
852         else:
853             return True
854
855     def getUserCacheFilename(self):
856         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
857         return home + "/.gitp4-usercache.txt"
858
859     def getUserMapFromPerforceServer(self):
860         if self.userMapFromPerforceServer:
861             return
862         self.users = {}
863         self.emails = {}
864
865         for output in p4CmdList("users"):
866             if not output.has_key("User"):
867                 continue
868             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
869             self.emails[output["Email"]] = output["User"]
870
871
872         s = ''
873         for (key, val) in self.users.items():
874             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
875
876         open(self.getUserCacheFilename(), "wb").write(s)
877         self.userMapFromPerforceServer = True
878
879     def loadUserMapFromCache(self):
880         self.users = {}
881         self.userMapFromPerforceServer = False
882         try:
883             cache = open(self.getUserCacheFilename(), "rb")
884             lines = cache.readlines()
885             cache.close()
886             for line in lines:
887                 entry = line.strip().split("\t")
888                 self.users[entry[0]] = entry[1]
889         except IOError:
890             self.getUserMapFromPerforceServer()
891
892 class P4Debug(Command):
893     def __init__(self):
894         Command.__init__(self)
895         self.options = []
896         self.description = "A tool to debug the output of p4 -G."
897         self.needsGit = False
898
899     def run(self, args):
900         j = 0
901         for output in p4CmdList(args):
902             print 'Element: %d' % j
903             j += 1
904             print output
905         return True
906
907 class P4RollBack(Command):
908     def __init__(self):
909         Command.__init__(self)
910         self.options = [
911             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
912         ]
913         self.description = "A tool to debug the multi-branch import. Don't use :)"
914         self.rollbackLocalBranches = False
915
916     def run(self, args):
917         if len(args) != 1:
918             return False
919         maxChange = int(args[0])
920
921         if "p4ExitCode" in p4Cmd("changes -m 1"):
922             die("Problems executing p4");
923
924         if self.rollbackLocalBranches:
925             refPrefix = "refs/heads/"
926             lines = read_pipe_lines("git rev-parse --symbolic --branches")
927         else:
928             refPrefix = "refs/remotes/"
929             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
930
931         for line in lines:
932             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
933                 line = line.strip()
934                 ref = refPrefix + line
935                 log = extractLogMessageFromGitCommit(ref)
936                 settings = extractSettingsGitLog(log)
937
938                 depotPaths = settings['depot-paths']
939                 change = settings['change']
940
941                 changed = False
942
943                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
944                                                            for p in depotPaths]))) == 0:
945                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
946                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
947                     continue
948
949                 while change and int(change) > maxChange:
950                     changed = True
951                     if self.verbose:
952                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
953                     system("git update-ref %s \"%s^\"" % (ref, ref))
954                     log = extractLogMessageFromGitCommit(ref)
955                     settings =  extractSettingsGitLog(log)
956
957
958                     depotPaths = settings['depot-paths']
959                     change = settings['change']
960
961                 if changed:
962                     print "%s rewound to %s" % (ref, change)
963
964         return True
965
966 class P4Submit(Command, P4UserMap):
967
968     conflict_behavior_choices = ("ask", "skip", "quit")
969
970     def __init__(self):
971         Command.__init__(self)
972         P4UserMap.__init__(self)
973         self.options = [
974                 optparse.make_option("--origin", dest="origin"),
975                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
976                 # preserve the user, requires relevant p4 permissions
977                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
978                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
979                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
980                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
981                 optparse.make_option("--conflict", dest="conflict_behavior",
982                                      choices=self.conflict_behavior_choices),
983                 optparse.make_option("--branch", dest="branch"),
984         ]
985         self.description = "Submit changes from git to the perforce depot."
986         self.usage += " [name of git branch to submit into perforce depot]"
987         self.origin = ""
988         self.detectRenames = False
989         self.preserveUser = gitConfigBool("git-p4.preserveUser")
990         self.dry_run = False
991         self.prepare_p4_only = False
992         self.conflict_behavior = None
993         self.isWindows = (platform.system() == "Windows")
994         self.exportLabels = False
995         self.p4HasMoveCommand = p4_has_move_command()
996         self.branch = None
997
998     def check(self):
999         if len(p4CmdList("opened ...")) > 0:
1000             die("You have files opened with perforce! Close them before starting the sync.")
1001
1002     def separate_jobs_from_description(self, message):
1003         """Extract and return a possible Jobs field in the commit
1004            message.  It goes into a separate section in the p4 change
1005            specification.
1006
1007            A jobs line starts with "Jobs:" and looks like a new field
1008            in a form.  Values are white-space separated on the same
1009            line or on following lines that start with a tab.
1010
1011            This does not parse and extract the full git commit message
1012            like a p4 form.  It just sees the Jobs: line as a marker
1013            to pass everything from then on directly into the p4 form,
1014            but outside the description section.
1015
1016            Return a tuple (stripped log message, jobs string)."""
1017
1018         m = re.search(r'^Jobs:', message, re.MULTILINE)
1019         if m is None:
1020             return (message, None)
1021
1022         jobtext = message[m.start():]
1023         stripped_message = message[:m.start()].rstrip()
1024         return (stripped_message, jobtext)
1025
1026     def prepareLogMessage(self, template, message, jobs):
1027         """Edits the template returned from "p4 change -o" to insert
1028            the message in the Description field, and the jobs text in
1029            the Jobs field."""
1030         result = ""
1031
1032         inDescriptionSection = False
1033
1034         for line in template.split("\n"):
1035             if line.startswith("#"):
1036                 result += line + "\n"
1037                 continue
1038
1039             if inDescriptionSection:
1040                 if line.startswith("Files:") or line.startswith("Jobs:"):
1041                     inDescriptionSection = False
1042                     # insert Jobs section
1043                     if jobs:
1044                         result += jobs + "\n"
1045                 else:
1046                     continue
1047             else:
1048                 if line.startswith("Description:"):
1049                     inDescriptionSection = True
1050                     line += "\n"
1051                     for messageLine in message.split("\n"):
1052                         line += "\t" + messageLine + "\n"
1053
1054             result += line + "\n"
1055
1056         return result
1057
1058     def patchRCSKeywords(self, file, pattern):
1059         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1060         (handle, outFileName) = tempfile.mkstemp(dir='.')
1061         try:
1062             outFile = os.fdopen(handle, "w+")
1063             inFile = open(file, "r")
1064             regexp = re.compile(pattern, re.VERBOSE)
1065             for line in inFile.readlines():
1066                 line = regexp.sub(r'$\1$', line)
1067                 outFile.write(line)
1068             inFile.close()
1069             outFile.close()
1070             # Forcibly overwrite the original file
1071             os.unlink(file)
1072             shutil.move(outFileName, file)
1073         except:
1074             # cleanup our temporary file
1075             os.unlink(outFileName)
1076             print "Failed to strip RCS keywords in %s" % file
1077             raise
1078
1079         print "Patched up RCS keywords in %s" % file
1080
1081     def p4UserForCommit(self,id):
1082         # Return the tuple (perforce user,git email) for a given git commit id
1083         self.getUserMapFromPerforceServer()
1084         gitEmail = read_pipe(["git", "log", "--max-count=1",
1085                               "--format=%ae", id])
1086         gitEmail = gitEmail.strip()
1087         if not self.emails.has_key(gitEmail):
1088             return (None,gitEmail)
1089         else:
1090             return (self.emails[gitEmail],gitEmail)
1091
1092     def checkValidP4Users(self,commits):
1093         # check if any git authors cannot be mapped to p4 users
1094         for id in commits:
1095             (user,email) = self.p4UserForCommit(id)
1096             if not user:
1097                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1098                 if gitConfigBool("git-p4.allowMissingP4Users"):
1099                     print "%s" % msg
1100                 else:
1101                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1102
1103     def lastP4Changelist(self):
1104         # Get back the last changelist number submitted in this client spec. This
1105         # then gets used to patch up the username in the change. If the same
1106         # client spec is being used by multiple processes then this might go
1107         # wrong.
1108         results = p4CmdList("client -o")        # find the current client
1109         client = None
1110         for r in results:
1111             if r.has_key('Client'):
1112                 client = r['Client']
1113                 break
1114         if not client:
1115             die("could not get client spec")
1116         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1117         for r in results:
1118             if r.has_key('change'):
1119                 return r['change']
1120         die("Could not get changelist number for last submit - cannot patch up user details")
1121
1122     def modifyChangelistUser(self, changelist, newUser):
1123         # fixup the user field of a changelist after it has been submitted.
1124         changes = p4CmdList("change -o %s" % changelist)
1125         if len(changes) != 1:
1126             die("Bad output from p4 change modifying %s to user %s" %
1127                 (changelist, newUser))
1128
1129         c = changes[0]
1130         if c['User'] == newUser: return   # nothing to do
1131         c['User'] = newUser
1132         input = marshal.dumps(c)
1133
1134         result = p4CmdList("change -f -i", stdin=input)
1135         for r in result:
1136             if r.has_key('code'):
1137                 if r['code'] == 'error':
1138                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1139             if r.has_key('data'):
1140                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1141                 return
1142         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1143
1144     def canChangeChangelists(self):
1145         # check to see if we have p4 admin or super-user permissions, either of
1146         # which are required to modify changelists.
1147         results = p4CmdList(["protects", self.depotPath])
1148         for r in results:
1149             if r.has_key('perm'):
1150                 if r['perm'] == 'admin':
1151                     return 1
1152                 if r['perm'] == 'super':
1153                     return 1
1154         return 0
1155
1156     def prepareSubmitTemplate(self):
1157         """Run "p4 change -o" to grab a change specification template.
1158            This does not use "p4 -G", as it is nice to keep the submission
1159            template in original order, since a human might edit it.
1160
1161            Remove lines in the Files section that show changes to files
1162            outside the depot path we're committing into."""
1163
1164         template = ""
1165         inFilesSection = False
1166         for line in p4_read_pipe_lines(['change', '-o']):
1167             if line.endswith("\r\n"):
1168                 line = line[:-2] + "\n"
1169             if inFilesSection:
1170                 if line.startswith("\t"):
1171                     # path starts and ends with a tab
1172                     path = line[1:]
1173                     lastTab = path.rfind("\t")
1174                     if lastTab != -1:
1175                         path = path[:lastTab]
1176                         if not p4PathStartsWith(path, self.depotPath):
1177                             continue
1178                 else:
1179                     inFilesSection = False
1180             else:
1181                 if line.startswith("Files:"):
1182                     inFilesSection = True
1183
1184             template += line
1185
1186         return template
1187
1188     def edit_template(self, template_file):
1189         """Invoke the editor to let the user change the submission
1190            message.  Return true if okay to continue with the submit."""
1191
1192         # if configured to skip the editing part, just submit
1193         if gitConfigBool("git-p4.skipSubmitEdit"):
1194             return True
1195
1196         # look at the modification time, to check later if the user saved
1197         # the file
1198         mtime = os.stat(template_file).st_mtime
1199
1200         # invoke the editor
1201         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1202             editor = os.environ.get("P4EDITOR")
1203         else:
1204             editor = read_pipe("git var GIT_EDITOR").strip()
1205         system(editor + " " + template_file)
1206
1207         # If the file was not saved, prompt to see if this patch should
1208         # be skipped.  But skip this verification step if configured so.
1209         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1210             return True
1211
1212         # modification time updated means user saved the file
1213         if os.stat(template_file).st_mtime > mtime:
1214             return True
1215
1216         while True:
1217             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1218             if response == 'y':
1219                 return True
1220             if response == 'n':
1221                 return False
1222
1223     def applyCommit(self, id):
1224         """Apply one commit, return True if it succeeded."""
1225
1226         print "Applying", read_pipe(["git", "show", "-s",
1227                                      "--format=format:%h %s", id])
1228
1229         (p4User, gitEmail) = self.p4UserForCommit(id)
1230
1231         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1232         filesToAdd = set()
1233         filesToDelete = set()
1234         editedFiles = set()
1235         pureRenameCopy = set()
1236         filesToChangeExecBit = {}
1237
1238         for line in diff:
1239             diff = parseDiffTreeEntry(line)
1240             modifier = diff['status']
1241             path = diff['src']
1242             if modifier == "M":
1243                 p4_edit(path)
1244                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1245                     filesToChangeExecBit[path] = diff['dst_mode']
1246                 editedFiles.add(path)
1247             elif modifier == "A":
1248                 filesToAdd.add(path)
1249                 filesToChangeExecBit[path] = diff['dst_mode']
1250                 if path in filesToDelete:
1251                     filesToDelete.remove(path)
1252             elif modifier == "D":
1253                 filesToDelete.add(path)
1254                 if path in filesToAdd:
1255                     filesToAdd.remove(path)
1256             elif modifier == "C":
1257                 src, dest = diff['src'], diff['dst']
1258                 p4_integrate(src, dest)
1259                 pureRenameCopy.add(dest)
1260                 if diff['src_sha1'] != diff['dst_sha1']:
1261                     p4_edit(dest)
1262                     pureRenameCopy.discard(dest)
1263                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1264                     p4_edit(dest)
1265                     pureRenameCopy.discard(dest)
1266                     filesToChangeExecBit[dest] = diff['dst_mode']
1267                 if self.isWindows:
1268                     # turn off read-only attribute
1269                     os.chmod(dest, stat.S_IWRITE)
1270                 os.unlink(dest)
1271                 editedFiles.add(dest)
1272             elif modifier == "R":
1273                 src, dest = diff['src'], diff['dst']
1274                 if self.p4HasMoveCommand:
1275                     p4_edit(src)        # src must be open before move
1276                     p4_move(src, dest)  # opens for (move/delete, move/add)
1277                 else:
1278                     p4_integrate(src, dest)
1279                     if diff['src_sha1'] != diff['dst_sha1']:
1280                         p4_edit(dest)
1281                     else:
1282                         pureRenameCopy.add(dest)
1283                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1284                     if not self.p4HasMoveCommand:
1285                         p4_edit(dest)   # with move: already open, writable
1286                     filesToChangeExecBit[dest] = diff['dst_mode']
1287                 if not self.p4HasMoveCommand:
1288                     if self.isWindows:
1289                         os.chmod(dest, stat.S_IWRITE)
1290                     os.unlink(dest)
1291                     filesToDelete.add(src)
1292                 editedFiles.add(dest)
1293             else:
1294                 die("unknown modifier %s for %s" % (modifier, path))
1295
1296         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1297         patchcmd = diffcmd + " | git apply "
1298         tryPatchCmd = patchcmd + "--check -"
1299         applyPatchCmd = patchcmd + "--check --apply -"
1300         patch_succeeded = True
1301
1302         if os.system(tryPatchCmd) != 0:
1303             fixed_rcs_keywords = False
1304             patch_succeeded = False
1305             print "Unfortunately applying the change failed!"
1306
1307             # Patch failed, maybe it's just RCS keyword woes. Look through
1308             # the patch to see if that's possible.
1309             if gitConfigBool("git-p4.attemptRCSCleanup"):
1310                 file = None
1311                 pattern = None
1312                 kwfiles = {}
1313                 for file in editedFiles | filesToDelete:
1314                     # did this file's delta contain RCS keywords?
1315                     pattern = p4_keywords_regexp_for_file(file)
1316
1317                     if pattern:
1318                         # this file is a possibility...look for RCS keywords.
1319                         regexp = re.compile(pattern, re.VERBOSE)
1320                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1321                             if regexp.search(line):
1322                                 if verbose:
1323                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1324                                 kwfiles[file] = pattern
1325                                 break
1326
1327                 for file in kwfiles:
1328                     if verbose:
1329                         print "zapping %s with %s" % (line,pattern)
1330                     # File is being deleted, so not open in p4.  Must
1331                     # disable the read-only bit on windows.
1332                     if self.isWindows and file not in editedFiles:
1333                         os.chmod(file, stat.S_IWRITE)
1334                     self.patchRCSKeywords(file, kwfiles[file])
1335                     fixed_rcs_keywords = True
1336
1337             if fixed_rcs_keywords:
1338                 print "Retrying the patch with RCS keywords cleaned up"
1339                 if os.system(tryPatchCmd) == 0:
1340                     patch_succeeded = True
1341
1342         if not patch_succeeded:
1343             for f in editedFiles:
1344                 p4_revert(f)
1345             return False
1346
1347         #
1348         # Apply the patch for real, and do add/delete/+x handling.
1349         #
1350         system(applyPatchCmd)
1351
1352         for f in filesToAdd:
1353             p4_add(f)
1354         for f in filesToDelete:
1355             p4_revert(f)
1356             p4_delete(f)
1357
1358         # Set/clear executable bits
1359         for f in filesToChangeExecBit.keys():
1360             mode = filesToChangeExecBit[f]
1361             setP4ExecBit(f, mode)
1362
1363         #
1364         # Build p4 change description, starting with the contents
1365         # of the git commit message.
1366         #
1367         logMessage = extractLogMessageFromGitCommit(id)
1368         logMessage = logMessage.strip()
1369         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1370
1371         template = self.prepareSubmitTemplate()
1372         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1373
1374         if self.preserveUser:
1375            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1376
1377         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1378             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1379             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1380             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1381
1382         separatorLine = "######## everything below this line is just the diff #######\n"
1383
1384         # diff
1385         if os.environ.has_key("P4DIFF"):
1386             del(os.environ["P4DIFF"])
1387         diff = ""
1388         for editedFile in editedFiles:
1389             diff += p4_read_pipe(['diff', '-du',
1390                                   wildcard_encode(editedFile)])
1391
1392         # new file diff
1393         newdiff = ""
1394         for newFile in filesToAdd:
1395             newdiff += "==== new file ====\n"
1396             newdiff += "--- /dev/null\n"
1397             newdiff += "+++ %s\n" % newFile
1398             f = open(newFile, "r")
1399             for line in f.readlines():
1400                 newdiff += "+" + line
1401             f.close()
1402
1403         # change description file: submitTemplate, separatorLine, diff, newdiff
1404         (handle, fileName) = tempfile.mkstemp()
1405         tmpFile = os.fdopen(handle, "w+")
1406         if self.isWindows:
1407             submitTemplate = submitTemplate.replace("\n", "\r\n")
1408             separatorLine = separatorLine.replace("\n", "\r\n")
1409             newdiff = newdiff.replace("\n", "\r\n")
1410         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1411         tmpFile.close()
1412
1413         if self.prepare_p4_only:
1414             #
1415             # Leave the p4 tree prepared, and the submit template around
1416             # and let the user decide what to do next
1417             #
1418             print
1419             print "P4 workspace prepared for submission."
1420             print "To submit or revert, go to client workspace"
1421             print "  " + self.clientPath
1422             print
1423             print "To submit, use \"p4 submit\" to write a new description,"
1424             print "or \"p4 submit -i %s\" to use the one prepared by" \
1425                   " \"git p4\"." % fileName
1426             print "You can delete the file \"%s\" when finished." % fileName
1427
1428             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1429                 print "To preserve change ownership by user %s, you must\n" \
1430                       "do \"p4 change -f <change>\" after submitting and\n" \
1431                       "edit the User field."
1432             if pureRenameCopy:
1433                 print "After submitting, renamed files must be re-synced."
1434                 print "Invoke \"p4 sync -f\" on each of these files:"
1435                 for f in pureRenameCopy:
1436                     print "  " + f
1437
1438             print
1439             print "To revert the changes, use \"p4 revert ...\", and delete"
1440             print "the submit template file \"%s\"" % fileName
1441             if filesToAdd:
1442                 print "Since the commit adds new files, they must be deleted:"
1443                 for f in filesToAdd:
1444                     print "  " + f
1445             print
1446             return True
1447
1448         #
1449         # Let the user edit the change description, then submit it.
1450         #
1451         if self.edit_template(fileName):
1452             # read the edited message and submit
1453             ret = True
1454             tmpFile = open(fileName, "rb")
1455             message = tmpFile.read()
1456             tmpFile.close()
1457             submitTemplate = message[:message.index(separatorLine)]
1458             if self.isWindows:
1459                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1460             p4_write_pipe(['submit', '-i'], submitTemplate)
1461
1462             if self.preserveUser:
1463                 if p4User:
1464                     # Get last changelist number. Cannot easily get it from
1465                     # the submit command output as the output is
1466                     # unmarshalled.
1467                     changelist = self.lastP4Changelist()
1468                     self.modifyChangelistUser(changelist, p4User)
1469
1470             # The rename/copy happened by applying a patch that created a
1471             # new file.  This leaves it writable, which confuses p4.
1472             for f in pureRenameCopy:
1473                 p4_sync(f, "-f")
1474
1475         else:
1476             # skip this patch
1477             ret = False
1478             print "Submission cancelled, undoing p4 changes."
1479             for f in editedFiles:
1480                 p4_revert(f)
1481             for f in filesToAdd:
1482                 p4_revert(f)
1483                 os.remove(f)
1484             for f in filesToDelete:
1485                 p4_revert(f)
1486
1487         os.remove(fileName)
1488         return ret
1489
1490     # Export git tags as p4 labels. Create a p4 label and then tag
1491     # with that.
1492     def exportGitTags(self, gitTags):
1493         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1494         if len(validLabelRegexp) == 0:
1495             validLabelRegexp = defaultLabelRegexp
1496         m = re.compile(validLabelRegexp)
1497
1498         for name in gitTags:
1499
1500             if not m.match(name):
1501                 if verbose:
1502                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1503                 continue
1504
1505             # Get the p4 commit this corresponds to
1506             logMessage = extractLogMessageFromGitCommit(name)
1507             values = extractSettingsGitLog(logMessage)
1508
1509             if not values.has_key('change'):
1510                 # a tag pointing to something not sent to p4; ignore
1511                 if verbose:
1512                     print "git tag %s does not give a p4 commit" % name
1513                 continue
1514             else:
1515                 changelist = values['change']
1516
1517             # Get the tag details.
1518             inHeader = True
1519             isAnnotated = False
1520             body = []
1521             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1522                 l = l.strip()
1523                 if inHeader:
1524                     if re.match(r'tag\s+', l):
1525                         isAnnotated = True
1526                     elif re.match(r'\s*$', l):
1527                         inHeader = False
1528                         continue
1529                 else:
1530                     body.append(l)
1531
1532             if not isAnnotated:
1533                 body = ["lightweight tag imported by git p4\n"]
1534
1535             # Create the label - use the same view as the client spec we are using
1536             clientSpec = getClientSpec()
1537
1538             labelTemplate  = "Label: %s\n" % name
1539             labelTemplate += "Description:\n"
1540             for b in body:
1541                 labelTemplate += "\t" + b + "\n"
1542             labelTemplate += "View:\n"
1543             for mapping in clientSpec.mappings:
1544                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1545
1546             if self.dry_run:
1547                 print "Would create p4 label %s for tag" % name
1548             elif self.prepare_p4_only:
1549                 print "Not creating p4 label %s for tag due to option" \
1550                       " --prepare-p4-only" % name
1551             else:
1552                 p4_write_pipe(["label", "-i"], labelTemplate)
1553
1554                 # Use the label
1555                 p4_system(["tag", "-l", name] +
1556                           ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1557
1558                 if verbose:
1559                     print "created p4 label for tag %s" % name
1560
1561     def run(self, args):
1562         if len(args) == 0:
1563             self.master = currentGitBranch()
1564             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1565                 die("Detecting current git branch failed!")
1566         elif len(args) == 1:
1567             self.master = args[0]
1568             if not branchExists(self.master):
1569                 die("Branch %s does not exist" % self.master)
1570         else:
1571             return False
1572
1573         allowSubmit = gitConfig("git-p4.allowSubmit")
1574         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1575             die("%s is not in git-p4.allowSubmit" % self.master)
1576
1577         [upstream, settings] = findUpstreamBranchPoint()
1578         self.depotPath = settings['depot-paths'][0]
1579         if len(self.origin) == 0:
1580             self.origin = upstream
1581
1582         if self.preserveUser:
1583             if not self.canChangeChangelists():
1584                 die("Cannot preserve user names without p4 super-user or admin permissions")
1585
1586         # if not set from the command line, try the config file
1587         if self.conflict_behavior is None:
1588             val = gitConfig("git-p4.conflict")
1589             if val:
1590                 if val not in self.conflict_behavior_choices:
1591                     die("Invalid value '%s' for config git-p4.conflict" % val)
1592             else:
1593                 val = "ask"
1594             self.conflict_behavior = val
1595
1596         if self.verbose:
1597             print "Origin branch is " + self.origin
1598
1599         if len(self.depotPath) == 0:
1600             print "Internal error: cannot locate perforce depot path from existing branches"
1601             sys.exit(128)
1602
1603         self.useClientSpec = False
1604         if gitConfigBool("git-p4.useclientspec"):
1605             self.useClientSpec = True
1606         if self.useClientSpec:
1607             self.clientSpecDirs = getClientSpec()
1608
1609         if self.useClientSpec:
1610             # all files are relative to the client spec
1611             self.clientPath = getClientRoot()
1612         else:
1613             self.clientPath = p4Where(self.depotPath)
1614
1615         if self.clientPath == "":
1616             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1617
1618         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1619         self.oldWorkingDirectory = os.getcwd()
1620
1621         # ensure the clientPath exists
1622         new_client_dir = False
1623         if not os.path.exists(self.clientPath):
1624             new_client_dir = True
1625             os.makedirs(self.clientPath)
1626
1627         chdir(self.clientPath)
1628         if self.dry_run:
1629             print "Would synchronize p4 checkout in %s" % self.clientPath
1630         else:
1631             print "Synchronizing p4 checkout..."
1632             if new_client_dir:
1633                 # old one was destroyed, and maybe nobody told p4
1634                 p4_sync("...", "-f")
1635             else:
1636                 p4_sync("...")
1637         self.check()
1638
1639         commits = []
1640         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1641             commits.append(line.strip())
1642         commits.reverse()
1643
1644         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1645             self.checkAuthorship = False
1646         else:
1647             self.checkAuthorship = True
1648
1649         if self.preserveUser:
1650             self.checkValidP4Users(commits)
1651
1652         #
1653         # Build up a set of options to be passed to diff when
1654         # submitting each commit to p4.
1655         #
1656         if self.detectRenames:
1657             # command-line -M arg
1658             self.diffOpts = "-M"
1659         else:
1660             # If not explicitly set check the config variable
1661             detectRenames = gitConfig("git-p4.detectRenames")
1662
1663             if detectRenames.lower() == "false" or detectRenames == "":
1664                 self.diffOpts = ""
1665             elif detectRenames.lower() == "true":
1666                 self.diffOpts = "-M"
1667             else:
1668                 self.diffOpts = "-M%s" % detectRenames
1669
1670         # no command-line arg for -C or --find-copies-harder, just
1671         # config variables
1672         detectCopies = gitConfig("git-p4.detectCopies")
1673         if detectCopies.lower() == "false" or detectCopies == "":
1674             pass
1675         elif detectCopies.lower() == "true":
1676             self.diffOpts += " -C"
1677         else:
1678             self.diffOpts += " -C%s" % detectCopies
1679
1680         if gitConfigBool("git-p4.detectCopiesHarder"):
1681             self.diffOpts += " --find-copies-harder"
1682
1683         #
1684         # Apply the commits, one at a time.  On failure, ask if should
1685         # continue to try the rest of the patches, or quit.
1686         #
1687         if self.dry_run:
1688             print "Would apply"
1689         applied = []
1690         last = len(commits) - 1
1691         for i, commit in enumerate(commits):
1692             if self.dry_run:
1693                 print " ", read_pipe(["git", "show", "-s",
1694                                       "--format=format:%h %s", commit])
1695                 ok = True
1696             else:
1697                 ok = self.applyCommit(commit)
1698             if ok:
1699                 applied.append(commit)
1700             else:
1701                 if self.prepare_p4_only and i < last:
1702                     print "Processing only the first commit due to option" \
1703                           " --prepare-p4-only"
1704                     break
1705                 if i < last:
1706                     quit = False
1707                     while True:
1708                         # prompt for what to do, or use the option/variable
1709                         if self.conflict_behavior == "ask":
1710                             print "What do you want to do?"
1711                             response = raw_input("[s]kip this commit but apply"
1712                                                  " the rest, or [q]uit? ")
1713                             if not response:
1714                                 continue
1715                         elif self.conflict_behavior == "skip":
1716                             response = "s"
1717                         elif self.conflict_behavior == "quit":
1718                             response = "q"
1719                         else:
1720                             die("Unknown conflict_behavior '%s'" %
1721                                 self.conflict_behavior)
1722
1723                         if response[0] == "s":
1724                             print "Skipping this commit, but applying the rest"
1725                             break
1726                         if response[0] == "q":
1727                             print "Quitting"
1728                             quit = True
1729                             break
1730                     if quit:
1731                         break
1732
1733         chdir(self.oldWorkingDirectory)
1734
1735         if self.dry_run:
1736             pass
1737         elif self.prepare_p4_only:
1738             pass
1739         elif len(commits) == len(applied):
1740             print "All commits applied!"
1741
1742             sync = P4Sync()
1743             if self.branch:
1744                 sync.branch = self.branch
1745             sync.run([])
1746
1747             rebase = P4Rebase()
1748             rebase.rebase()
1749
1750         else:
1751             if len(applied) == 0:
1752                 print "No commits applied."
1753             else:
1754                 print "Applied only the commits marked with '*':"
1755                 for c in commits:
1756                     if c in applied:
1757                         star = "*"
1758                     else:
1759                         star = " "
1760                     print star, read_pipe(["git", "show", "-s",
1761                                            "--format=format:%h %s",  c])
1762                 print "You will have to do 'git p4 sync' and rebase."
1763
1764         if gitConfigBool("git-p4.exportLabels"):
1765             self.exportLabels = True
1766
1767         if self.exportLabels:
1768             p4Labels = getP4Labels(self.depotPath)
1769             gitTags = getGitTags()
1770
1771             missingGitTags = gitTags - p4Labels
1772             self.exportGitTags(missingGitTags)
1773
1774         # exit with error unless everything applied perfecly
1775         if len(commits) != len(applied):
1776                 sys.exit(1)
1777
1778         return True
1779
1780 class View(object):
1781     """Represent a p4 view ("p4 help views"), and map files in a
1782        repo according to the view."""
1783
1784     class Path(object):
1785         """A depot or client path, possibly containing wildcards.
1786            The only one supported is ... at the end, currently.
1787            Initialize with the full path, with //depot or //client."""
1788
1789         def __init__(self, path, is_depot):
1790             self.path = path
1791             self.is_depot = is_depot
1792             self.find_wildcards()
1793             # remember the prefix bit, useful for relative mappings
1794             m = re.match("(//[^/]+/)", self.path)
1795             if not m:
1796                 die("Path %s does not start with //prefix/" % self.path)
1797             prefix = m.group(1)
1798             if not self.is_depot:
1799                 # strip //client/ on client paths
1800                 self.path = self.path[len(prefix):]
1801
1802         def find_wildcards(self):
1803             """Make sure wildcards are valid, and set up internal
1804                variables."""
1805
1806             self.ends_triple_dot = False
1807             # There are three wildcards allowed in p4 views
1808             # (see "p4 help views").  This code knows how to
1809             # handle "..." (only at the end), but cannot deal with
1810             # "%%n" or "*".  Only check the depot_side, as p4 should
1811             # validate that the client_side matches too.
1812             if re.search(r'%%[1-9]', self.path):
1813                 die("Can't handle %%n wildcards in view: %s" % self.path)
1814             if self.path.find("*") >= 0:
1815                 die("Can't handle * wildcards in view: %s" % self.path)
1816             triple_dot_index = self.path.find("...")
1817             if triple_dot_index >= 0:
1818                 if triple_dot_index != len(self.path) - 3:
1819                     die("Can handle only single ... wildcard, at end: %s" %
1820                         self.path)
1821                 self.ends_triple_dot = True
1822
1823         def ensure_compatible(self, other_path):
1824             """Make sure the wildcards agree."""
1825             if self.ends_triple_dot != other_path.ends_triple_dot:
1826                  die("Both paths must end with ... if either does;\n" +
1827                      "paths: %s %s" % (self.path, other_path.path))
1828
1829         def match_wildcards(self, test_path):
1830             """See if this test_path matches us, and fill in the value
1831                of the wildcards if so.  Returns a tuple of
1832                (True|False, wildcards[]).  For now, only the ... at end
1833                is supported, so at most one wildcard."""
1834             if self.ends_triple_dot:
1835                 dotless = self.path[:-3]
1836                 if test_path.startswith(dotless):
1837                     wildcard = test_path[len(dotless):]
1838                     return (True, [ wildcard ])
1839             else:
1840                 if test_path == self.path:
1841                     return (True, [])
1842             return (False, [])
1843
1844         def match(self, test_path):
1845             """Just return if it matches; don't bother with the wildcards."""
1846             b, _ = self.match_wildcards(test_path)
1847             return b
1848
1849         def fill_in_wildcards(self, wildcards):
1850             """Return the relative path, with the wildcards filled in
1851                if there are any."""
1852             if self.ends_triple_dot:
1853                 return self.path[:-3] + wildcards[0]
1854             else:
1855                 return self.path
1856
1857     class Mapping(object):
1858         def __init__(self, depot_side, client_side, overlay, exclude):
1859             # depot_side is without the trailing /... if it had one
1860             self.depot_side = View.Path(depot_side, is_depot=True)
1861             self.client_side = View.Path(client_side, is_depot=False)
1862             self.overlay = overlay  # started with "+"
1863             self.exclude = exclude  # started with "-"
1864             assert not (self.overlay and self.exclude)
1865             self.depot_side.ensure_compatible(self.client_side)
1866
1867         def __str__(self):
1868             c = " "
1869             if self.overlay:
1870                 c = "+"
1871             if self.exclude:
1872                 c = "-"
1873             return "View.Mapping: %s%s -> %s" % \
1874                    (c, self.depot_side.path, self.client_side.path)
1875
1876         def map_depot_to_client(self, depot_path):
1877             """Calculate the client path if using this mapping on the
1878                given depot path; does not consider the effect of other
1879                mappings in a view.  Even excluded mappings are returned."""
1880             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1881             if not matches:
1882                 return ""
1883             client_path = self.client_side.fill_in_wildcards(wildcards)
1884             return client_path
1885
1886     #
1887     # View methods
1888     #
1889     def __init__(self):
1890         self.mappings = []
1891
1892     def append(self, view_line):
1893         """Parse a view line, splitting it into depot and client
1894            sides.  Append to self.mappings, preserving order."""
1895
1896         # Split the view line into exactly two words.  P4 enforces
1897         # structure on these lines that simplifies this quite a bit.
1898         #
1899         # Either or both words may be double-quoted.
1900         # Single quotes do not matter.
1901         # Double-quote marks cannot occur inside the words.
1902         # A + or - prefix is also inside the quotes.
1903         # There are no quotes unless they contain a space.
1904         # The line is already white-space stripped.
1905         # The two words are separated by a single space.
1906         #
1907         if view_line[0] == '"':
1908             # First word is double quoted.  Find its end.
1909             close_quote_index = view_line.find('"', 1)
1910             if close_quote_index <= 0:
1911                 die("No first-word closing quote found: %s" % view_line)
1912             depot_side = view_line[1:close_quote_index]
1913             # skip closing quote and space
1914             rhs_index = close_quote_index + 1 + 1
1915         else:
1916             space_index = view_line.find(" ")
1917             if space_index <= 0:
1918                 die("No word-splitting space found: %s" % view_line)
1919             depot_side = view_line[0:space_index]
1920             rhs_index = space_index + 1
1921
1922         if view_line[rhs_index] == '"':
1923             # Second word is double quoted.  Make sure there is a
1924             # double quote at the end too.
1925             if not view_line.endswith('"'):
1926                 die("View line with rhs quote should end with one: %s" %
1927                     view_line)
1928             # skip the quotes
1929             client_side = view_line[rhs_index+1:-1]
1930         else:
1931             client_side = view_line[rhs_index:]
1932
1933         # prefix + means overlay on previous mapping
1934         overlay = False
1935         if depot_side.startswith("+"):
1936             overlay = True
1937             depot_side = depot_side[1:]
1938
1939         # prefix - means exclude this path
1940         exclude = False
1941         if depot_side.startswith("-"):
1942             exclude = True
1943             depot_side = depot_side[1:]
1944
1945         m = View.Mapping(depot_side, client_side, overlay, exclude)
1946         self.mappings.append(m)
1947
1948     def map_in_client(self, depot_path):
1949         """Return the relative location in the client where this
1950            depot file should live.  Returns "" if the file should
1951            not be mapped in the client."""
1952
1953         paths_filled = []
1954         client_path = ""
1955
1956         # look at later entries first
1957         for m in self.mappings[::-1]:
1958
1959             # see where will this path end up in the client
1960             p = m.map_depot_to_client(depot_path)
1961
1962             if p == "":
1963                 # Depot path does not belong in client.  Must remember
1964                 # this, as previous items should not cause files to
1965                 # exist in this path either.  Remember that the list is
1966                 # being walked from the end, which has higher precedence.
1967                 # Overlap mappings do not exclude previous mappings.
1968                 if not m.overlay:
1969                     paths_filled.append(m.client_side)
1970
1971             else:
1972                 # This mapping matched; no need to search any further.
1973                 # But, the mapping could be rejected if the client path
1974                 # has already been claimed by an earlier mapping (i.e.
1975                 # one later in the list, which we are walking backwards).
1976                 already_mapped_in_client = False
1977                 for f in paths_filled:
1978                     # this is View.Path.match
1979                     if f.match(p):
1980                         already_mapped_in_client = True
1981                         break
1982                 if not already_mapped_in_client:
1983                     # Include this file, unless it is from a line that
1984                     # explicitly said to exclude it.
1985                     if not m.exclude:
1986                         client_path = p
1987
1988                 # a match, even if rejected, always stops the search
1989                 break
1990
1991         return client_path
1992
1993 class P4Sync(Command, P4UserMap):
1994     delete_actions = ( "delete", "move/delete", "purge" )
1995
1996     def __init__(self):
1997         Command.__init__(self)
1998         P4UserMap.__init__(self)
1999         self.options = [
2000                 optparse.make_option("--branch", dest="branch"),
2001                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2002                 optparse.make_option("--changesfile", dest="changesFile"),
2003                 optparse.make_option("--silent", dest="silent", action="store_true"),
2004                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2005                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2006                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2007                                      help="Import into refs/heads/ , not refs/remotes"),
2008                 optparse.make_option("--max-changes", dest="maxChanges"),
2009                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2010                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2011                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2012                                      help="Only sync files that are included in the Perforce Client Spec")
2013         ]
2014         self.description = """Imports from Perforce into a git repository.\n
2015     example:
2016     //depot/my/project/ -- to import the current head
2017     //depot/my/project/@all -- to import everything
2018     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2019
2020     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2021
2022         self.usage += " //depot/path[@revRange]"
2023         self.silent = False
2024         self.createdBranches = set()
2025         self.committedChanges = set()
2026         self.branch = ""
2027         self.detectBranches = False
2028         self.detectLabels = False
2029         self.importLabels = False
2030         self.changesFile = ""
2031         self.syncWithOrigin = True
2032         self.importIntoRemotes = True
2033         self.maxChanges = ""
2034         self.keepRepoPath = False
2035         self.depotPaths = None
2036         self.p4BranchesInGit = []
2037         self.cloneExclude = []
2038         self.useClientSpec = False
2039         self.useClientSpec_from_options = False
2040         self.clientSpecDirs = None
2041         self.tempBranches = []
2042         self.tempBranchLocation = "git-p4-tmp"
2043
2044         if gitConfig("git-p4.syncFromOrigin") == "false":
2045             self.syncWithOrigin = False
2046
2047     # Force a checkpoint in fast-import and wait for it to finish
2048     def checkpoint(self):
2049         self.gitStream.write("checkpoint\n\n")
2050         self.gitStream.write("progress checkpoint\n\n")
2051         out = self.gitOutput.readline()
2052         if self.verbose:
2053             print "checkpoint finished: " + out
2054
2055     def extractFilesFromCommit(self, commit):
2056         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2057                              for path in self.cloneExclude]
2058         files = []
2059         fnum = 0
2060         while commit.has_key("depotFile%s" % fnum):
2061             path =  commit["depotFile%s" % fnum]
2062
2063             if [p for p in self.cloneExclude
2064                 if p4PathStartsWith(path, p)]:
2065                 found = False
2066             else:
2067                 found = [p for p in self.depotPaths
2068                          if p4PathStartsWith(path, p)]
2069             if not found:
2070                 fnum = fnum + 1
2071                 continue
2072
2073             file = {}
2074             file["path"] = path
2075             file["rev"] = commit["rev%s" % fnum]
2076             file["action"] = commit["action%s" % fnum]
2077             file["type"] = commit["type%s" % fnum]
2078             files.append(file)
2079             fnum = fnum + 1
2080         return files
2081
2082     def stripRepoPath(self, path, prefixes):
2083         """When streaming files, this is called to map a p4 depot path
2084            to where it should go in git.  The prefixes are either
2085            self.depotPaths, or self.branchPrefixes in the case of
2086            branch detection."""
2087
2088         if self.useClientSpec:
2089             # branch detection moves files up a level (the branch name)
2090             # from what client spec interpretation gives
2091             path = self.clientSpecDirs.map_in_client(path)
2092             if self.detectBranches:
2093                 for b in self.knownBranches:
2094                     if path.startswith(b + "/"):
2095                         path = path[len(b)+1:]
2096
2097         elif self.keepRepoPath:
2098             # Preserve everything in relative path name except leading
2099             # //depot/; just look at first prefix as they all should
2100             # be in the same depot.
2101             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2102             if p4PathStartsWith(path, depot):
2103                 path = path[len(depot):]
2104
2105         else:
2106             for p in prefixes:
2107                 if p4PathStartsWith(path, p):
2108                     path = path[len(p):]
2109                     break
2110
2111         path = wildcard_decode(path)
2112         return path
2113
2114     def splitFilesIntoBranches(self, commit):
2115         """Look at each depotFile in the commit to figure out to what
2116            branch it belongs."""
2117
2118         branches = {}
2119         fnum = 0
2120         while commit.has_key("depotFile%s" % fnum):
2121             path =  commit["depotFile%s" % fnum]
2122             found = [p for p in self.depotPaths
2123                      if p4PathStartsWith(path, p)]
2124             if not found:
2125                 fnum = fnum + 1
2126                 continue
2127
2128             file = {}
2129             file["path"] = path
2130             file["rev"] = commit["rev%s" % fnum]
2131             file["action"] = commit["action%s" % fnum]
2132             file["type"] = commit["type%s" % fnum]
2133             fnum = fnum + 1
2134
2135             # start with the full relative path where this file would
2136             # go in a p4 client
2137             if self.useClientSpec:
2138                 relPath = self.clientSpecDirs.map_in_client(path)
2139             else:
2140                 relPath = self.stripRepoPath(path, self.depotPaths)
2141
2142             for branch in self.knownBranches.keys():
2143                 # add a trailing slash so that a commit into qt/4.2foo
2144                 # doesn't end up in qt/4.2, e.g.
2145                 if relPath.startswith(branch + "/"):
2146                     if branch not in branches:
2147                         branches[branch] = []
2148                     branches[branch].append(file)
2149                     break
2150
2151         return branches
2152
2153     # output one file from the P4 stream
2154     # - helper for streamP4Files
2155
2156     def streamOneP4File(self, file, contents):
2157         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2158         if verbose:
2159             sys.stderr.write("%s\n" % relPath)
2160
2161         (type_base, type_mods) = split_p4_type(file["type"])
2162
2163         git_mode = "100644"
2164         if "x" in type_mods:
2165             git_mode = "100755"
2166         if type_base == "symlink":
2167             git_mode = "120000"
2168             # p4 print on a symlink contains "target\n"; remove the newline
2169             data = ''.join(contents)
2170             contents = [data[:-1]]
2171
2172         if type_base == "utf16":
2173             # p4 delivers different text in the python output to -G
2174             # than it does when using "print -o", or normal p4 client
2175             # operations.  utf16 is converted to ascii or utf8, perhaps.
2176             # But ascii text saved as -t utf16 is completely mangled.
2177             # Invoke print -o to get the real contents.
2178             #
2179             # On windows, the newlines will always be mangled by print, so put
2180             # them back too.  This is not needed to the cygwin windows version,
2181             # just the native "NT" type.
2182             #
2183             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2184             if p4_version_string().find("/NT") >= 0:
2185                 text = text.replace("\r\n", "\n")
2186             contents = [ text ]
2187
2188         if type_base == "apple":
2189             # Apple filetype files will be streamed as a concatenation of
2190             # its appledouble header and the contents.  This is useless
2191             # on both macs and non-macs.  If using "print -q -o xx", it
2192             # will create "xx" with the data, and "%xx" with the header.
2193             # This is also not very useful.
2194             #
2195             # Ideally, someday, this script can learn how to generate
2196             # appledouble files directly and import those to git, but
2197             # non-mac machines can never find a use for apple filetype.
2198             print "\nIgnoring apple filetype file %s" % file['depotFile']
2199             return
2200
2201         # Note that we do not try to de-mangle keywords on utf16 files,
2202         # even though in theory somebody may want that.
2203         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2204         if pattern:
2205             regexp = re.compile(pattern, re.VERBOSE)
2206             text = ''.join(contents)
2207             text = regexp.sub(r'$\1$', text)
2208             contents = [ text ]
2209
2210         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2211
2212         # total length...
2213         length = 0
2214         for d in contents:
2215             length = length + len(d)
2216
2217         self.gitStream.write("data %d\n" % length)
2218         for d in contents:
2219             self.gitStream.write(d)
2220         self.gitStream.write("\n")
2221
2222     def streamOneP4Deletion(self, file):
2223         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2224         if verbose:
2225             sys.stderr.write("delete %s\n" % relPath)
2226         self.gitStream.write("D %s\n" % relPath)
2227
2228     # handle another chunk of streaming data
2229     def streamP4FilesCb(self, marshalled):
2230
2231         # catch p4 errors and complain
2232         err = None
2233         if "code" in marshalled:
2234             if marshalled["code"] == "error":
2235                 if "data" in marshalled:
2236                     err = marshalled["data"].rstrip()
2237         if err:
2238             f = None
2239             if self.stream_have_file_info:
2240                 if "depotFile" in self.stream_file:
2241                     f = self.stream_file["depotFile"]
2242             # force a failure in fast-import, else an empty
2243             # commit will be made
2244             self.gitStream.write("\n")
2245             self.gitStream.write("die-now\n")
2246             self.gitStream.close()
2247             # ignore errors, but make sure it exits first
2248             self.importProcess.wait()
2249             if f:
2250                 die("Error from p4 print for %s: %s" % (f, err))
2251             else:
2252                 die("Error from p4 print: %s" % err)
2253
2254         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2255             # start of a new file - output the old one first
2256             self.streamOneP4File(self.stream_file, self.stream_contents)
2257             self.stream_file = {}
2258             self.stream_contents = []
2259             self.stream_have_file_info = False
2260
2261         # pick up the new file information... for the
2262         # 'data' field we need to append to our array
2263         for k in marshalled.keys():
2264             if k == 'data':
2265                 self.stream_contents.append(marshalled['data'])
2266             else:
2267                 self.stream_file[k] = marshalled[k]
2268
2269         self.stream_have_file_info = True
2270
2271     # Stream directly from "p4 files" into "git fast-import"
2272     def streamP4Files(self, files):
2273         filesForCommit = []
2274         filesToRead = []
2275         filesToDelete = []
2276
2277         for f in files:
2278             # if using a client spec, only add the files that have
2279             # a path in the client
2280             if self.clientSpecDirs:
2281                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2282                     continue
2283
2284             filesForCommit.append(f)
2285             if f['action'] in self.delete_actions:
2286                 filesToDelete.append(f)
2287             else:
2288                 filesToRead.append(f)
2289
2290         # deleted files...
2291         for f in filesToDelete:
2292             self.streamOneP4Deletion(f)
2293
2294         if len(filesToRead) > 0:
2295             self.stream_file = {}
2296             self.stream_contents = []
2297             self.stream_have_file_info = False
2298
2299             # curry self argument
2300             def streamP4FilesCbSelf(entry):
2301                 self.streamP4FilesCb(entry)
2302
2303             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2304
2305             p4CmdList(["-x", "-", "print"],
2306                       stdin=fileArgs,
2307                       cb=streamP4FilesCbSelf)
2308
2309             # do the last chunk
2310             if self.stream_file.has_key('depotFile'):
2311                 self.streamOneP4File(self.stream_file, self.stream_contents)
2312
2313     def make_email(self, userid):
2314         if userid in self.users:
2315             return self.users[userid]
2316         else:
2317             return "%s <a@b>" % userid
2318
2319     # Stream a p4 tag
2320     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2321         if verbose:
2322             print "writing tag %s for commit %s" % (labelName, commit)
2323         gitStream.write("tag %s\n" % labelName)
2324         gitStream.write("from %s\n" % commit)
2325
2326         if labelDetails.has_key('Owner'):
2327             owner = labelDetails["Owner"]
2328         else:
2329             owner = None
2330
2331         # Try to use the owner of the p4 label, or failing that,
2332         # the current p4 user id.
2333         if owner:
2334             email = self.make_email(owner)
2335         else:
2336             email = self.make_email(self.p4UserId())
2337         tagger = "%s %s %s" % (email, epoch, self.tz)
2338
2339         gitStream.write("tagger %s\n" % tagger)
2340
2341         print "labelDetails=",labelDetails
2342         if labelDetails.has_key('Description'):
2343             description = labelDetails['Description']
2344         else:
2345             description = 'Label from git p4'
2346
2347         gitStream.write("data %d\n" % len(description))
2348         gitStream.write(description)
2349         gitStream.write("\n")
2350
2351     def commit(self, details, files, branch, parent = ""):
2352         epoch = details["time"]
2353         author = details["user"]
2354
2355         if self.verbose:
2356             print "commit into %s" % branch
2357
2358         # start with reading files; if that fails, we should not
2359         # create a commit.
2360         new_files = []
2361         for f in files:
2362             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2363                 new_files.append (f)
2364             else:
2365                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2366
2367         self.gitStream.write("commit %s\n" % branch)
2368 #        gitStream.write("mark :%s\n" % details["change"])
2369         self.committedChanges.add(int(details["change"]))
2370         committer = ""
2371         if author not in self.users:
2372             self.getUserMapFromPerforceServer()
2373         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2374
2375         self.gitStream.write("committer %s\n" % committer)
2376
2377         self.gitStream.write("data <<EOT\n")
2378         self.gitStream.write(details["desc"])
2379         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2380                              (','.join(self.branchPrefixes), details["change"]))
2381         if len(details['options']) > 0:
2382             self.gitStream.write(": options = %s" % details['options'])
2383         self.gitStream.write("]\nEOT\n\n")
2384
2385         if len(parent) > 0:
2386             if self.verbose:
2387                 print "parent %s" % parent
2388             self.gitStream.write("from %s\n" % parent)
2389
2390         self.streamP4Files(new_files)
2391         self.gitStream.write("\n")
2392
2393         change = int(details["change"])
2394
2395         if self.labels.has_key(change):
2396             label = self.labels[change]
2397             labelDetails = label[0]
2398             labelRevisions = label[1]
2399             if self.verbose:
2400                 print "Change %s is labelled %s" % (change, labelDetails)
2401
2402             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2403                                                 for p in self.branchPrefixes])
2404
2405             if len(files) == len(labelRevisions):
2406
2407                 cleanedFiles = {}
2408                 for info in files:
2409                     if info["action"] in self.delete_actions:
2410                         continue
2411                     cleanedFiles[info["depotFile"]] = info["rev"]
2412
2413                 if cleanedFiles == labelRevisions:
2414                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2415
2416                 else:
2417                     if not self.silent:
2418                         print ("Tag %s does not match with change %s: files do not match."
2419                                % (labelDetails["label"], change))
2420
2421             else:
2422                 if not self.silent:
2423                     print ("Tag %s does not match with change %s: file count is different."
2424                            % (labelDetails["label"], change))
2425
2426     # Build a dictionary of changelists and labels, for "detect-labels" option.
2427     def getLabels(self):
2428         self.labels = {}
2429
2430         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2431         if len(l) > 0 and not self.silent:
2432             print "Finding files belonging to labels in %s" % `self.depotPaths`
2433
2434         for output in l:
2435             label = output["label"]
2436             revisions = {}
2437             newestChange = 0
2438             if self.verbose:
2439                 print "Querying files for label %s" % label
2440             for file in p4CmdList(["files"] +
2441                                       ["%s...@%s" % (p, label)
2442                                           for p in self.depotPaths]):
2443                 revisions[file["depotFile"]] = file["rev"]
2444                 change = int(file["change"])
2445                 if change > newestChange:
2446                     newestChange = change
2447
2448             self.labels[newestChange] = [output, revisions]
2449
2450         if self.verbose:
2451             print "Label changes: %s" % self.labels.keys()
2452
2453     # Import p4 labels as git tags. A direct mapping does not
2454     # exist, so assume that if all the files are at the same revision
2455     # then we can use that, or it's something more complicated we should
2456     # just ignore.
2457     def importP4Labels(self, stream, p4Labels):
2458         if verbose:
2459             print "import p4 labels: " + ' '.join(p4Labels)
2460
2461         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2462         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2463         if len(validLabelRegexp) == 0:
2464             validLabelRegexp = defaultLabelRegexp
2465         m = re.compile(validLabelRegexp)
2466
2467         for name in p4Labels:
2468             commitFound = False
2469
2470             if not m.match(name):
2471                 if verbose:
2472                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2473                 continue
2474
2475             if name in ignoredP4Labels:
2476                 continue
2477
2478             labelDetails = p4CmdList(['label', "-o", name])[0]
2479
2480             # get the most recent changelist for each file in this label
2481             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2482                                 for p in self.depotPaths])
2483
2484             if change.has_key('change'):
2485                 # find the corresponding git commit; take the oldest commit
2486                 changelist = int(change['change'])
2487                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2488                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2489                 if len(gitCommit) == 0:
2490                     print "could not find git commit for changelist %d" % changelist
2491                 else:
2492                     gitCommit = gitCommit.strip()
2493                     commitFound = True
2494                     # Convert from p4 time format
2495                     try:
2496                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2497                     except ValueError:
2498                         print "Could not convert label time %s" % labelDetails['Update']
2499                         tmwhen = 1
2500
2501                     when = int(time.mktime(tmwhen))
2502                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2503                     if verbose:
2504                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2505             else:
2506                 if verbose:
2507                     print "Label %s has no changelists - possibly deleted?" % name
2508
2509             if not commitFound:
2510                 # We can't import this label; don't try again as it will get very
2511                 # expensive repeatedly fetching all the files for labels that will
2512                 # never be imported. If the label is moved in the future, the
2513                 # ignore will need to be removed manually.
2514                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2515
2516     def guessProjectName(self):
2517         for p in self.depotPaths:
2518             if p.endswith("/"):
2519                 p = p[:-1]
2520             p = p[p.strip().rfind("/") + 1:]
2521             if not p.endswith("/"):
2522                p += "/"
2523             return p
2524
2525     def getBranchMapping(self):
2526         lostAndFoundBranches = set()
2527
2528         user = gitConfig("git-p4.branchUser")
2529         if len(user) > 0:
2530             command = "branches -u %s" % user
2531         else:
2532             command = "branches"
2533
2534         for info in p4CmdList(command):
2535             details = p4Cmd(["branch", "-o", info["branch"]])
2536             viewIdx = 0
2537             while details.has_key("View%s" % viewIdx):
2538                 paths = details["View%s" % viewIdx].split(" ")
2539                 viewIdx = viewIdx + 1
2540                 # require standard //depot/foo/... //depot/bar/... mapping
2541                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2542                     continue
2543                 source = paths[0]
2544                 destination = paths[1]
2545                 ## HACK
2546                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2547                     source = source[len(self.depotPaths[0]):-4]
2548                     destination = destination[len(self.depotPaths[0]):-4]
2549
2550                     if destination in self.knownBranches:
2551                         if not self.silent:
2552                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2553                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2554                         continue
2555
2556                     self.knownBranches[destination] = source
2557
2558                     lostAndFoundBranches.discard(destination)
2559
2560                     if source not in self.knownBranches:
2561                         lostAndFoundBranches.add(source)
2562
2563         # Perforce does not strictly require branches to be defined, so we also
2564         # check git config for a branch list.
2565         #
2566         # Example of branch definition in git config file:
2567         # [git-p4]
2568         #   branchList=main:branchA
2569         #   branchList=main:branchB
2570         #   branchList=branchA:branchC
2571         configBranches = gitConfigList("git-p4.branchList")
2572         for branch in configBranches:
2573             if branch:
2574                 (source, destination) = branch.split(":")
2575                 self.knownBranches[destination] = source
2576
2577                 lostAndFoundBranches.discard(destination)
2578
2579                 if source not in self.knownBranches:
2580                     lostAndFoundBranches.add(source)
2581
2582
2583         for branch in lostAndFoundBranches:
2584             self.knownBranches[branch] = branch
2585
2586     def getBranchMappingFromGitBranches(self):
2587         branches = p4BranchesInGit(self.importIntoRemotes)
2588         for branch in branches.keys():
2589             if branch == "master":
2590                 branch = "main"
2591             else:
2592                 branch = branch[len(self.projectName):]
2593             self.knownBranches[branch] = branch
2594
2595     def updateOptionDict(self, d):
2596         option_keys = {}
2597         if self.keepRepoPath:
2598             option_keys['keepRepoPath'] = 1
2599
2600         d["options"] = ' '.join(sorted(option_keys.keys()))
2601
2602     def readOptions(self, d):
2603         self.keepRepoPath = (d.has_key('options')
2604                              and ('keepRepoPath' in d['options']))
2605
2606     def gitRefForBranch(self, branch):
2607         if branch == "main":
2608             return self.refPrefix + "master"
2609
2610         if len(branch) <= 0:
2611             return branch
2612
2613         return self.refPrefix + self.projectName + branch
2614
2615     def gitCommitByP4Change(self, ref, change):
2616         if self.verbose:
2617             print "looking in ref " + ref + " for change %s using bisect..." % change
2618
2619         earliestCommit = ""
2620         latestCommit = parseRevision(ref)
2621
2622         while True:
2623             if self.verbose:
2624                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2625             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2626             if len(next) == 0:
2627                 if self.verbose:
2628                     print "argh"
2629                 return ""
2630             log = extractLogMessageFromGitCommit(next)
2631             settings = extractSettingsGitLog(log)
2632             currentChange = int(settings['change'])
2633             if self.verbose:
2634                 print "current change %s" % currentChange
2635
2636             if currentChange == change:
2637                 if self.verbose:
2638                     print "found %s" % next
2639                 return next
2640
2641             if currentChange < change:
2642                 earliestCommit = "^%s" % next
2643             else:
2644                 latestCommit = "%s" % next
2645
2646         return ""
2647
2648     def importNewBranch(self, branch, maxChange):
2649         # make fast-import flush all changes to disk and update the refs using the checkpoint
2650         # command so that we can try to find the branch parent in the git history
2651         self.gitStream.write("checkpoint\n\n");
2652         self.gitStream.flush();
2653         branchPrefix = self.depotPaths[0] + branch + "/"
2654         range = "@1,%s" % maxChange
2655         #print "prefix" + branchPrefix
2656         changes = p4ChangesForPaths([branchPrefix], range)
2657         if len(changes) <= 0:
2658             return False
2659         firstChange = changes[0]
2660         #print "first change in branch: %s" % firstChange
2661         sourceBranch = self.knownBranches[branch]
2662         sourceDepotPath = self.depotPaths[0] + sourceBranch
2663         sourceRef = self.gitRefForBranch(sourceBranch)
2664         #print "source " + sourceBranch
2665
2666         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2667         #print "branch parent: %s" % branchParentChange
2668         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2669         if len(gitParent) > 0:
2670             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2671             #print "parent git commit: %s" % gitParent
2672
2673         self.importChanges(changes)
2674         return True
2675
2676     def searchParent(self, parent, branch, target):
2677         parentFound = False
2678         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2679                                      "--no-merges", parent]):
2680             blob = blob.strip()
2681             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2682                 parentFound = True
2683                 if self.verbose:
2684                     print "Found parent of %s in commit %s" % (branch, blob)
2685                 break
2686         if parentFound:
2687             return blob
2688         else:
2689             return None
2690
2691     def importChanges(self, changes):
2692         cnt = 1
2693         for change in changes:
2694             description = p4_describe(change)
2695             self.updateOptionDict(description)
2696
2697             if not self.silent:
2698                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2699                 sys.stdout.flush()
2700             cnt = cnt + 1
2701
2702             try:
2703                 if self.detectBranches:
2704                     branches = self.splitFilesIntoBranches(description)
2705                     for branch in branches.keys():
2706                         ## HACK  --hwn
2707                         branchPrefix = self.depotPaths[0] + branch + "/"
2708                         self.branchPrefixes = [ branchPrefix ]
2709
2710                         parent = ""
2711
2712                         filesForCommit = branches[branch]
2713
2714                         if self.verbose:
2715                             print "branch is %s" % branch
2716
2717                         self.updatedBranches.add(branch)
2718
2719                         if branch not in self.createdBranches:
2720                             self.createdBranches.add(branch)
2721                             parent = self.knownBranches[branch]
2722                             if parent == branch:
2723                                 parent = ""
2724                             else:
2725                                 fullBranch = self.projectName + branch
2726                                 if fullBranch not in self.p4BranchesInGit:
2727                                     if not self.silent:
2728                                         print("\n    Importing new branch %s" % fullBranch);
2729                                     if self.importNewBranch(branch, change - 1):
2730                                         parent = ""
2731                                         self.p4BranchesInGit.append(fullBranch)
2732                                     if not self.silent:
2733                                         print("\n    Resuming with change %s" % change);
2734
2735                                 if self.verbose:
2736                                     print "parent determined through known branches: %s" % parent
2737
2738                         branch = self.gitRefForBranch(branch)
2739                         parent = self.gitRefForBranch(parent)
2740
2741                         if self.verbose:
2742                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2743
2744                         if len(parent) == 0 and branch in self.initialParents:
2745                             parent = self.initialParents[branch]
2746                             del self.initialParents[branch]
2747
2748                         blob = None
2749                         if len(parent) > 0:
2750                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2751                             if self.verbose:
2752                                 print "Creating temporary branch: " + tempBranch
2753                             self.commit(description, filesForCommit, tempBranch)
2754                             self.tempBranches.append(tempBranch)
2755                             self.checkpoint()
2756                             blob = self.searchParent(parent, branch, tempBranch)
2757                         if blob:
2758                             self.commit(description, filesForCommit, branch, blob)
2759                         else:
2760                             if self.verbose:
2761                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2762                             self.commit(description, filesForCommit, branch, parent)
2763                 else:
2764                     files = self.extractFilesFromCommit(description)
2765                     self.commit(description, files, self.branch,
2766                                 self.initialParent)
2767                     # only needed once, to connect to the previous commit
2768                     self.initialParent = ""
2769             except IOError:
2770                 print self.gitError.read()
2771                 sys.exit(1)
2772
2773     def importHeadRevision(self, revision):
2774         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2775
2776         details = {}
2777         details["user"] = "git perforce import user"
2778         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2779                            % (' '.join(self.depotPaths), revision))
2780         details["change"] = revision
2781         newestRevision = 0
2782
2783         fileCnt = 0
2784         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2785
2786         for info in p4CmdList(["files"] + fileArgs):
2787
2788             if 'code' in info and info['code'] == 'error':
2789                 sys.stderr.write("p4 returned an error: %s\n"
2790                                  % info['data'])
2791                 if info['data'].find("must refer to client") >= 0:
2792                     sys.stderr.write("This particular p4 error is misleading.\n")
2793                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2794                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2795                 sys.exit(1)
2796             if 'p4ExitCode' in info:
2797                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2798                 sys.exit(1)
2799
2800
2801             change = int(info["change"])
2802             if change > newestRevision:
2803                 newestRevision = change
2804
2805             if info["action"] in self.delete_actions:
2806                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2807                 #fileCnt = fileCnt + 1
2808                 continue
2809
2810             for prop in ["depotFile", "rev", "action", "type" ]:
2811                 details["%s%s" % (prop, fileCnt)] = info[prop]
2812
2813             fileCnt = fileCnt + 1
2814
2815         details["change"] = newestRevision
2816
2817         # Use time from top-most change so that all git p4 clones of
2818         # the same p4 repo have the same commit SHA1s.
2819         res = p4_describe(newestRevision)
2820         details["time"] = res["time"]
2821
2822         self.updateOptionDict(details)
2823         try:
2824             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2825         except IOError:
2826             print "IO error with git fast-import. Is your git version recent enough?"
2827             print self.gitError.read()
2828
2829
2830     def run(self, args):
2831         self.depotPaths = []
2832         self.changeRange = ""
2833         self.previousDepotPaths = []
2834         self.hasOrigin = False
2835
2836         # map from branch depot path to parent branch
2837         self.knownBranches = {}
2838         self.initialParents = {}
2839
2840         if self.importIntoRemotes:
2841             self.refPrefix = "refs/remotes/p4/"
2842         else:
2843             self.refPrefix = "refs/heads/p4/"
2844
2845         if self.syncWithOrigin:
2846             self.hasOrigin = originP4BranchesExist()
2847             if self.hasOrigin:
2848                 if not self.silent:
2849                     print 'Syncing with origin first, using "git fetch origin"'
2850                 system("git fetch origin")
2851
2852         branch_arg_given = bool(self.branch)
2853         if len(self.branch) == 0:
2854             self.branch = self.refPrefix + "master"
2855             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2856                 system("git update-ref %s refs/heads/p4" % self.branch)
2857                 system("git branch -D p4")
2858
2859         # accept either the command-line option, or the configuration variable
2860         if self.useClientSpec:
2861             # will use this after clone to set the variable
2862             self.useClientSpec_from_options = True
2863         else:
2864             if gitConfigBool("git-p4.useclientspec"):
2865                 self.useClientSpec = True
2866         if self.useClientSpec:
2867             self.clientSpecDirs = getClientSpec()
2868
2869         # TODO: should always look at previous commits,
2870         # merge with previous imports, if possible.
2871         if args == []:
2872             if self.hasOrigin:
2873                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2874
2875             # branches holds mapping from branch name to sha1
2876             branches = p4BranchesInGit(self.importIntoRemotes)
2877
2878             # restrict to just this one, disabling detect-branches
2879             if branch_arg_given:
2880                 short = self.branch.split("/")[-1]
2881                 if short in branches:
2882                     self.p4BranchesInGit = [ short ]
2883             else:
2884                 self.p4BranchesInGit = branches.keys()
2885
2886             if len(self.p4BranchesInGit) > 1:
2887                 if not self.silent:
2888                     print "Importing from/into multiple branches"
2889                 self.detectBranches = True
2890                 for branch in branches.keys():
2891                     self.initialParents[self.refPrefix + branch] = \
2892                         branches[branch]
2893
2894             if self.verbose:
2895                 print "branches: %s" % self.p4BranchesInGit
2896
2897             p4Change = 0
2898             for branch in self.p4BranchesInGit:
2899                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2900
2901                 settings = extractSettingsGitLog(logMsg)
2902
2903                 self.readOptions(settings)
2904                 if (settings.has_key('depot-paths')
2905                     and settings.has_key ('change')):
2906                     change = int(settings['change']) + 1
2907                     p4Change = max(p4Change, change)
2908
2909                     depotPaths = sorted(settings['depot-paths'])
2910                     if self.previousDepotPaths == []:
2911                         self.previousDepotPaths = depotPaths
2912                     else:
2913                         paths = []
2914                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2915                             prev_list = prev.split("/")
2916                             cur_list = cur.split("/")
2917                             for i in range(0, min(len(cur_list), len(prev_list))):
2918                                 if cur_list[i] <> prev_list[i]:
2919                                     i = i - 1
2920                                     break
2921
2922                             paths.append ("/".join(cur_list[:i + 1]))
2923
2924                         self.previousDepotPaths = paths
2925
2926             if p4Change > 0:
2927                 self.depotPaths = sorted(self.previousDepotPaths)
2928                 self.changeRange = "@%s,#head" % p4Change
2929                 if not self.silent and not self.detectBranches:
2930                     print "Performing incremental import into %s git branch" % self.branch
2931
2932         # accept multiple ref name abbreviations:
2933         #    refs/foo/bar/branch -> use it exactly
2934         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2935         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2936         if not self.branch.startswith("refs/"):
2937             if self.importIntoRemotes:
2938                 prepend = "refs/remotes/"
2939             else:
2940                 prepend = "refs/heads/"
2941             if not self.branch.startswith("p4/"):
2942                 prepend += "p4/"
2943             self.branch = prepend + self.branch
2944
2945         if len(args) == 0 and self.depotPaths:
2946             if not self.silent:
2947                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2948         else:
2949             if self.depotPaths and self.depotPaths != args:
2950                 print ("previous import used depot path %s and now %s was specified. "
2951                        "This doesn't work!" % (' '.join (self.depotPaths),
2952                                                ' '.join (args)))
2953                 sys.exit(1)
2954
2955             self.depotPaths = sorted(args)
2956
2957         revision = ""
2958         self.users = {}
2959
2960         # Make sure no revision specifiers are used when --changesfile
2961         # is specified.
2962         bad_changesfile = False
2963         if len(self.changesFile) > 0:
2964             for p in self.depotPaths:
2965                 if p.find("@") >= 0 or p.find("#") >= 0:
2966                     bad_changesfile = True
2967                     break
2968         if bad_changesfile:
2969             die("Option --changesfile is incompatible with revision specifiers")
2970
2971         newPaths = []
2972         for p in self.depotPaths:
2973             if p.find("@") != -1:
2974                 atIdx = p.index("@")
2975                 self.changeRange = p[atIdx:]
2976                 if self.changeRange == "@all":
2977                     self.changeRange = ""
2978                 elif ',' not in self.changeRange:
2979                     revision = self.changeRange
2980                     self.changeRange = ""
2981                 p = p[:atIdx]
2982             elif p.find("#") != -1:
2983                 hashIdx = p.index("#")
2984                 revision = p[hashIdx:]
2985                 p = p[:hashIdx]
2986             elif self.previousDepotPaths == []:
2987                 # pay attention to changesfile, if given, else import
2988                 # the entire p4 tree at the head revision
2989                 if len(self.changesFile) == 0:
2990                     revision = "#head"
2991
2992             p = re.sub ("\.\.\.$", "", p)
2993             if not p.endswith("/"):
2994                 p += "/"
2995
2996             newPaths.append(p)
2997
2998         self.depotPaths = newPaths
2999
3000         # --detect-branches may change this for each branch
3001         self.branchPrefixes = self.depotPaths
3002
3003         self.loadUserMapFromCache()
3004         self.labels = {}
3005         if self.detectLabels:
3006             self.getLabels();
3007
3008         if self.detectBranches:
3009             ## FIXME - what's a P4 projectName ?
3010             self.projectName = self.guessProjectName()
3011
3012             if self.hasOrigin:
3013                 self.getBranchMappingFromGitBranches()
3014             else:
3015                 self.getBranchMapping()
3016             if self.verbose:
3017                 print "p4-git branches: %s" % self.p4BranchesInGit
3018                 print "initial parents: %s" % self.initialParents
3019             for b in self.p4BranchesInGit:
3020                 if b != "master":
3021
3022                     ## FIXME
3023                     b = b[len(self.projectName):]
3024                 self.createdBranches.add(b)
3025
3026         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3027
3028         self.importProcess = subprocess.Popen(["git", "fast-import"],
3029                                               stdin=subprocess.PIPE,
3030                                               stdout=subprocess.PIPE,
3031                                               stderr=subprocess.PIPE);
3032         self.gitOutput = self.importProcess.stdout
3033         self.gitStream = self.importProcess.stdin
3034         self.gitError = self.importProcess.stderr
3035
3036         if revision:
3037             self.importHeadRevision(revision)
3038         else:
3039             changes = []
3040
3041             if len(self.changesFile) > 0:
3042                 output = open(self.changesFile).readlines()
3043                 changeSet = set()
3044                 for line in output:
3045                     changeSet.add(int(line))
3046
3047                 for change in changeSet:
3048                     changes.append(change)
3049
3050                 changes.sort()
3051             else:
3052                 # catch "git p4 sync" with no new branches, in a repo that
3053                 # does not have any existing p4 branches
3054                 if len(args) == 0:
3055                     if not self.p4BranchesInGit:
3056                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3057
3058                     # The default branch is master, unless --branch is used to
3059                     # specify something else.  Make sure it exists, or complain
3060                     # nicely about how to use --branch.
3061                     if not self.detectBranches:
3062                         if not branch_exists(self.branch):
3063                             if branch_arg_given:
3064                                 die("Error: branch %s does not exist." % self.branch)
3065                             else:
3066                                 die("Error: no branch %s; perhaps specify one with --branch." %
3067                                     self.branch)
3068
3069                 if self.verbose:
3070                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3071                                                               self.changeRange)
3072                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
3073
3074                 if len(self.maxChanges) > 0:
3075                     changes = changes[:min(int(self.maxChanges), len(changes))]
3076
3077             if len(changes) == 0:
3078                 if not self.silent:
3079                     print "No changes to import!"
3080             else:
3081                 if not self.silent and not self.detectBranches:
3082                     print "Import destination: %s" % self.branch
3083
3084                 self.updatedBranches = set()
3085
3086                 if not self.detectBranches:
3087                     if args:
3088                         # start a new branch
3089                         self.initialParent = ""
3090                     else:
3091                         # build on a previous revision
3092                         self.initialParent = parseRevision(self.branch)
3093
3094                 self.importChanges(changes)
3095
3096                 if not self.silent:
3097                     print ""
3098                     if len(self.updatedBranches) > 0:
3099                         sys.stdout.write("Updated branches: ")
3100                         for b in self.updatedBranches:
3101                             sys.stdout.write("%s " % b)
3102                         sys.stdout.write("\n")
3103
3104         if gitConfigBool("git-p4.importLabels"):
3105             self.importLabels = True
3106
3107         if self.importLabels:
3108             p4Labels = getP4Labels(self.depotPaths)
3109             gitTags = getGitTags()
3110
3111             missingP4Labels = p4Labels - gitTags
3112             self.importP4Labels(self.gitStream, missingP4Labels)
3113
3114         self.gitStream.close()
3115         if self.importProcess.wait() != 0:
3116             die("fast-import failed: %s" % self.gitError.read())
3117         self.gitOutput.close()
3118         self.gitError.close()
3119
3120         # Cleanup temporary branches created during import
3121         if self.tempBranches != []:
3122             for branch in self.tempBranches:
3123                 read_pipe("git update-ref -d %s" % branch)
3124             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3125
3126         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3127         # a convenient shortcut refname "p4".
3128         if self.importIntoRemotes:
3129             head_ref = self.refPrefix + "HEAD"
3130             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3131                 system(["git", "symbolic-ref", head_ref, self.branch])
3132
3133         return True
3134
3135 class P4Rebase(Command):
3136     def __init__(self):
3137         Command.__init__(self)
3138         self.options = [
3139                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3140         ]
3141         self.importLabels = False
3142         self.description = ("Fetches the latest revision from perforce and "
3143                             + "rebases the current work (branch) against it")
3144
3145     def run(self, args):
3146         sync = P4Sync()
3147         sync.importLabels = self.importLabels
3148         sync.run([])
3149
3150         return self.rebase()
3151
3152     def rebase(self):
3153         if os.system("git update-index --refresh") != 0:
3154             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3155         if len(read_pipe("git diff-index HEAD --")) > 0:
3156             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3157
3158         [upstream, settings] = findUpstreamBranchPoint()
3159         if len(upstream) == 0:
3160             die("Cannot find upstream branchpoint for rebase")
3161
3162         # the branchpoint may be p4/foo~3, so strip off the parent
3163         upstream = re.sub("~[0-9]+$", "", upstream)
3164
3165         print "Rebasing the current branch onto %s" % upstream
3166         oldHead = read_pipe("git rev-parse HEAD").strip()
3167         system("git rebase %s" % upstream)
3168         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3169         return True
3170
3171 class P4Clone(P4Sync):
3172     def __init__(self):
3173         P4Sync.__init__(self)
3174         self.description = "Creates a new git repository and imports from Perforce into it"
3175         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3176         self.options += [
3177             optparse.make_option("--destination", dest="cloneDestination",
3178                                  action='store', default=None,
3179                                  help="where to leave result of the clone"),
3180             optparse.make_option("-/", dest="cloneExclude",
3181                                  action="append", type="string",
3182                                  help="exclude depot path"),
3183             optparse.make_option("--bare", dest="cloneBare",
3184                                  action="store_true", default=False),
3185         ]
3186         self.cloneDestination = None
3187         self.needsGit = False
3188         self.cloneBare = False
3189
3190     # This is required for the "append" cloneExclude action
3191     def ensure_value(self, attr, value):
3192         if not hasattr(self, attr) or getattr(self, attr) is None:
3193             setattr(self, attr, value)
3194         return getattr(self, attr)
3195
3196     def defaultDestination(self, args):
3197         ## TODO: use common prefix of args?
3198         depotPath = args[0]
3199         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3200         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3201         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3202         depotDir = re.sub(r"/$", "", depotDir)
3203         return os.path.split(depotDir)[1]
3204
3205     def run(self, args):
3206         if len(args) < 1:
3207             return False
3208
3209         if self.keepRepoPath and not self.cloneDestination:
3210             sys.stderr.write("Must specify destination for --keep-path\n")
3211             sys.exit(1)
3212
3213         depotPaths = args
3214
3215         if not self.cloneDestination and len(depotPaths) > 1:
3216             self.cloneDestination = depotPaths[-1]
3217             depotPaths = depotPaths[:-1]
3218
3219         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3220         for p in depotPaths:
3221             if not p.startswith("//"):
3222                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3223                 return False
3224
3225         if not self.cloneDestination:
3226             self.cloneDestination = self.defaultDestination(args)
3227
3228         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3229
3230         if not os.path.exists(self.cloneDestination):
3231             os.makedirs(self.cloneDestination)
3232         chdir(self.cloneDestination)
3233
3234         init_cmd = [ "git", "init" ]
3235         if self.cloneBare:
3236             init_cmd.append("--bare")
3237         retcode = subprocess.call(init_cmd)
3238         if retcode:
3239             raise CalledProcessError(retcode, init_cmd)
3240
3241         if not P4Sync.run(self, depotPaths):
3242             return False
3243
3244         # create a master branch and check out a work tree
3245         if gitBranchExists(self.branch):
3246             system([ "git", "branch", "master", self.branch ])
3247             if not self.cloneBare:
3248                 system([ "git", "checkout", "-f" ])
3249         else:
3250             print 'Not checking out any branch, use ' \
3251                   '"git checkout -q -b master <branch>"'
3252
3253         # auto-set this variable if invoked with --use-client-spec
3254         if self.useClientSpec_from_options:
3255             system("git config --bool git-p4.useclientspec true")
3256
3257         return True
3258
3259 class P4Branches(Command):
3260     def __init__(self):
3261         Command.__init__(self)
3262         self.options = [ ]
3263         self.description = ("Shows the git branches that hold imports and their "
3264                             + "corresponding perforce depot paths")
3265         self.verbose = False
3266
3267     def run(self, args):
3268         if originP4BranchesExist():
3269             createOrUpdateBranchesFromOrigin()
3270
3271         cmdline = "git rev-parse --symbolic "
3272         cmdline += " --remotes"
3273
3274         for line in read_pipe_lines(cmdline):
3275             line = line.strip()
3276
3277             if not line.startswith('p4/') or line == "p4/HEAD":
3278                 continue
3279             branch = line
3280
3281             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3282             settings = extractSettingsGitLog(log)
3283
3284             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3285         return True
3286
3287 class HelpFormatter(optparse.IndentedHelpFormatter):
3288     def __init__(self):
3289         optparse.IndentedHelpFormatter.__init__(self)
3290
3291     def format_description(self, description):
3292         if description:
3293             return description + "\n"
3294         else:
3295             return ""
3296
3297 def printUsage(commands):
3298     print "usage: %s <command> [options]" % sys.argv[0]
3299     print ""
3300     print "valid commands: %s" % ", ".join(commands)
3301     print ""
3302     print "Try %s <command> --help for command specific help." % sys.argv[0]
3303     print ""
3304
3305 commands = {
3306     "debug" : P4Debug,
3307     "submit" : P4Submit,
3308     "commit" : P4Submit,
3309     "sync" : P4Sync,
3310     "rebase" : P4Rebase,
3311     "clone" : P4Clone,
3312     "rollback" : P4RollBack,
3313     "branches" : P4Branches
3314 }
3315
3316
3317 def main():
3318     if len(sys.argv[1:]) == 0:
3319         printUsage(commands.keys())
3320         sys.exit(2)
3321
3322     cmdName = sys.argv[1]
3323     try:
3324         klass = commands[cmdName]
3325         cmd = klass()
3326     except KeyError:
3327         print "unknown command %s" % cmdName
3328         print ""
3329         printUsage(commands.keys())
3330         sys.exit(2)
3331
3332     options = cmd.options
3333     cmd.gitdir = os.environ.get("GIT_DIR", None)
3334
3335     args = sys.argv[2:]
3336
3337     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3338     if cmd.needsGit:
3339         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3340
3341     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3342                                    options,
3343                                    description = cmd.description,
3344                                    formatter = HelpFormatter())
3345
3346     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3347     global verbose
3348     verbose = cmd.verbose
3349     if cmd.needsGit:
3350         if cmd.gitdir == None:
3351             cmd.gitdir = os.path.abspath(".git")
3352             if not isValidGitDir(cmd.gitdir):
3353                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3354                 if os.path.exists(cmd.gitdir):
3355                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3356                     if len(cdup) > 0:
3357                         chdir(cdup);
3358
3359         if not isValidGitDir(cmd.gitdir):
3360             if isValidGitDir(cmd.gitdir + "/.git"):
3361                 cmd.gitdir += "/.git"
3362             else:
3363                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3364
3365         os.environ["GIT_DIR"] = cmd.gitdir
3366
3367     if not cmd.run(args):
3368         parser.print_help()
3369         sys.exit(2)
3370
3371
3372 if __name__ == '__main__':
3373     main()