Merge branch 'maint'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27     from subprocess import CalledProcessError
28 except ImportError:
29     # from python2.7:subprocess.py
30     # Exception classes used by this module.
31     class CalledProcessError(Exception):
32         """This exception is raised when a process run by check_call() returns
33         a non-zero exit status.  The exit status will be stored in the
34         returncode attribute."""
35         def __init__(self, returncode, cmd):
36             self.returncode = returncode
37             self.cmd = cmd
38         def __str__(self):
39             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 def p4_build_cmd(cmd):
47     """Build a suitable p4 command line.
48
49     This consolidates building and returning a p4 command line into one
50     location. It means that hooking into the environment, or other configuration
51     can be done more easily.
52     """
53     real_cmd = ["p4"]
54
55     user = gitConfig("git-p4.user")
56     if len(user) > 0:
57         real_cmd += ["-u",user]
58
59     password = gitConfig("git-p4.password")
60     if len(password) > 0:
61         real_cmd += ["-P", password]
62
63     port = gitConfig("git-p4.port")
64     if len(port) > 0:
65         real_cmd += ["-p", port]
66
67     host = gitConfig("git-p4.host")
68     if len(host) > 0:
69         real_cmd += ["-H", host]
70
71     client = gitConfig("git-p4.client")
72     if len(client) > 0:
73         real_cmd += ["-c", client]
74
75
76     if isinstance(cmd,basestring):
77         real_cmd = ' '.join(real_cmd) + ' ' + cmd
78     else:
79         real_cmd += cmd
80     return real_cmd
81
82 def chdir(path, is_client_path=False):
83     """Do chdir to the given path, and set the PWD environment
84        variable for use by P4.  It does not look at getcwd() output.
85        Since we're not using the shell, it is necessary to set the
86        PWD environment variable explicitly.
87
88        Normally, expand the path to force it to be absolute.  This
89        addresses the use of relative path names inside P4 settings,
90        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
91        as given; it looks for .p4config using PWD.
92
93        If is_client_path, the path was handed to us directly by p4,
94        and may be a symbolic link.  Do not call os.getcwd() in this
95        case, because it will cause p4 to think that PWD is not inside
96        the client path.
97        """
98
99     os.chdir(path)
100     if not is_client_path:
101         path = os.getcwd()
102     os.environ['PWD'] = path
103
104 def die(msg):
105     if verbose:
106         raise Exception(msg)
107     else:
108         sys.stderr.write(msg + "\n")
109         sys.exit(1)
110
111 def write_pipe(c, stdin):
112     if verbose:
113         sys.stderr.write('Writing pipe: %s\n' % str(c))
114
115     expand = isinstance(c,basestring)
116     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117     pipe = p.stdin
118     val = pipe.write(stdin)
119     pipe.close()
120     if p.wait():
121         die('Command failed: %s' % str(c))
122
123     return val
124
125 def p4_write_pipe(c, stdin):
126     real_cmd = p4_build_cmd(c)
127     return write_pipe(real_cmd, stdin)
128
129 def read_pipe(c, ignore_error=False):
130     if verbose:
131         sys.stderr.write('Reading pipe: %s\n' % str(c))
132
133     expand = isinstance(c,basestring)
134     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135     pipe = p.stdout
136     val = pipe.read()
137     if p.wait() and not ignore_error:
138         die('Command failed: %s' % str(c))
139
140     return val
141
142 def p4_read_pipe(c, ignore_error=False):
143     real_cmd = p4_build_cmd(c)
144     return read_pipe(real_cmd, ignore_error)
145
146 def read_pipe_lines(c):
147     if verbose:
148         sys.stderr.write('Reading pipe: %s\n' % str(c))
149
150     expand = isinstance(c, basestring)
151     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152     pipe = p.stdout
153     val = pipe.readlines()
154     if pipe.close() or p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_read_pipe_lines(c):
160     """Specifically invoke p4 on the command supplied. """
161     real_cmd = p4_build_cmd(c)
162     return read_pipe_lines(real_cmd)
163
164 def p4_has_command(cmd):
165     """Ask p4 for help on this command.  If it returns an error, the
166        command does not exist in this version of p4."""
167     real_cmd = p4_build_cmd(["help", cmd])
168     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169                                    stderr=subprocess.PIPE)
170     p.communicate()
171     return p.returncode == 0
172
173 def p4_has_move_command():
174     """See if the move command exists, that it supports -k, and that
175        it has not been administratively disabled.  The arguments
176        must be correct, but the filenames do not have to exist.  Use
177        ones with wildcards so even if they exist, it will fail."""
178
179     if not p4_has_command("move"):
180         return False
181     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183     (out, err) = p.communicate()
184     # return code will be 1 in either case
185     if err.find("Invalid option") >= 0:
186         return False
187     if err.find("disabled") >= 0:
188         return False
189     # assume it failed because @... was invalid changelist
190     return True
191
192 def system(cmd):
193     expand = isinstance(cmd,basestring)
194     if verbose:
195         sys.stderr.write("executing %s\n" % str(cmd))
196     retcode = subprocess.call(cmd, shell=expand)
197     if retcode:
198         raise CalledProcessError(retcode, cmd)
199
200 def p4_system(cmd):
201     """Specifically invoke p4 as the system command. """
202     real_cmd = p4_build_cmd(cmd)
203     expand = isinstance(real_cmd, basestring)
204     retcode = subprocess.call(real_cmd, shell=expand)
205     if retcode:
206         raise CalledProcessError(retcode, real_cmd)
207
208 _p4_version_string = None
209 def p4_version_string():
210     """Read the version string, showing just the last line, which
211        hopefully is the interesting version bit.
212
213        $ p4 -V
214        Perforce - The Fast Software Configuration Management System.
215        Copyright 1995-2011 Perforce Software.  All rights reserved.
216        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
217     """
218     global _p4_version_string
219     if not _p4_version_string:
220         a = p4_read_pipe_lines(["-V"])
221         _p4_version_string = a[-1].rstrip()
222     return _p4_version_string
223
224 def p4_integrate(src, dest):
225     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
226
227 def p4_sync(f, *options):
228     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
229
230 def p4_add(f):
231     # forcibly add file names with wildcards
232     if wildcard_present(f):
233         p4_system(["add", "-f", f])
234     else:
235         p4_system(["add", f])
236
237 def p4_delete(f):
238     p4_system(["delete", wildcard_encode(f)])
239
240 def p4_edit(f):
241     p4_system(["edit", wildcard_encode(f)])
242
243 def p4_revert(f):
244     p4_system(["revert", wildcard_encode(f)])
245
246 def p4_reopen(type, f):
247     p4_system(["reopen", "-t", type, wildcard_encode(f)])
248
249 def p4_move(src, dest):
250     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
251
252 def p4_describe(change):
253     """Make sure it returns a valid result by checking for
254        the presence of field "time".  Return a dict of the
255        results."""
256
257     ds = p4CmdList(["describe", "-s", str(change)])
258     if len(ds) != 1:
259         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
260
261     d = ds[0]
262
263     if "p4ExitCode" in d:
264         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265                                                       str(d)))
266     if "code" in d:
267         if d["code"] == "error":
268             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
269
270     if "time" not in d:
271         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
272
273     return d
274
275 #
276 # Canonicalize the p4 type and return a tuple of the
277 # base type, plus any modifiers.  See "p4 help filetypes"
278 # for a list and explanation.
279 #
280 def split_p4_type(p4type):
281
282     p4_filetypes_historical = {
283         "ctempobj": "binary+Sw",
284         "ctext": "text+C",
285         "cxtext": "text+Cx",
286         "ktext": "text+k",
287         "kxtext": "text+kx",
288         "ltext": "text+F",
289         "tempobj": "binary+FSw",
290         "ubinary": "binary+F",
291         "uresource": "resource+F",
292         "uxbinary": "binary+Fx",
293         "xbinary": "binary+x",
294         "xltext": "text+Fx",
295         "xtempobj": "binary+Swx",
296         "xtext": "text+x",
297         "xunicode": "unicode+x",
298         "xutf16": "utf16+x",
299     }
300     if p4type in p4_filetypes_historical:
301         p4type = p4_filetypes_historical[p4type]
302     mods = ""
303     s = p4type.split("+")
304     base = s[0]
305     mods = ""
306     if len(s) > 1:
307         mods = s[1]
308     return (base, mods)
309
310 #
311 # return the raw p4 type of a file (text, text+ko, etc)
312 #
313 def p4_type(file):
314     results = p4CmdList(["fstat", "-T", "headType", file])
315     return results[0]['headType']
316
317 #
318 # Given a type base and modifier, return a regexp matching
319 # the keywords that can be expanded in the file
320 #
321 def p4_keywords_regexp_for_type(base, type_mods):
322     if base in ("text", "unicode", "binary"):
323         kwords = None
324         if "ko" in type_mods:
325             kwords = 'Id|Header'
326         elif "k" in type_mods:
327             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328         else:
329             return None
330         pattern = r"""
331             \$              # Starts with a dollar, followed by...
332             (%s)            # one of the keywords, followed by...
333             (:[^$\n]+)?     # possibly an old expansion, followed by...
334             \$              # another dollar
335             """ % kwords
336         return pattern
337     else:
338         return None
339
340 #
341 # Given a file, return a regexp matching the possible
342 # RCS keywords that will be expanded, or None for files
343 # with kw expansion turned off.
344 #
345 def p4_keywords_regexp_for_file(file):
346     if not os.path.exists(file):
347         return None
348     else:
349         (type_base, type_mods) = split_p4_type(p4_type(file))
350         return p4_keywords_regexp_for_type(type_base, type_mods)
351
352 def setP4ExecBit(file, mode):
353     # Reopens an already open file and changes the execute bit to match
354     # the execute bit setting in the passed in mode.
355
356     p4Type = "+x"
357
358     if not isModeExec(mode):
359         p4Type = getP4OpenedType(file)
360         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362         if p4Type[-1] == "+":
363             p4Type = p4Type[0:-1]
364
365     p4_reopen(p4Type, file)
366
367 def getP4OpenedType(file):
368     # Returns the perforce file type for the given file.
369
370     result = p4_read_pipe(["opened", wildcard_encode(file)])
371     match = re.match(".*\((.+)\)\r?$", result)
372     if match:
373         return match.group(1)
374     else:
375         die("Could not determine file type for %s (result: '%s')" % (file, result))
376
377 # Return the set of all p4 labels
378 def getP4Labels(depotPaths):
379     labels = set()
380     if isinstance(depotPaths,basestring):
381         depotPaths = [depotPaths]
382
383     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384         label = l['label']
385         labels.add(label)
386
387     return labels
388
389 # Return the set of all git tags
390 def getGitTags():
391     gitTags = set()
392     for line in read_pipe_lines(["git", "tag"]):
393         tag = line.strip()
394         gitTags.add(tag)
395     return gitTags
396
397 def diffTreePattern():
398     # This is a simple generator for the diff tree regex pattern. This could be
399     # a class variable if this and parseDiffTreeEntry were a part of a class.
400     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401     while True:
402         yield pattern
403
404 def parseDiffTreeEntry(entry):
405     """Parses a single diff tree entry into its component elements.
406
407     See git-diff-tree(1) manpage for details about the format of the diff
408     output. This method returns a dictionary with the following elements:
409
410     src_mode - The mode of the source file
411     dst_mode - The mode of the destination file
412     src_sha1 - The sha1 for the source file
413     dst_sha1 - The sha1 fr the destination file
414     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415     status_score - The score for the status (applicable for 'C' and 'R'
416                    statuses). This is None if there is no score.
417     src - The path for the source file.
418     dst - The path for the destination file. This is only present for
419           copy or renames. If it is not present, this is None.
420
421     If the pattern is not matched, None is returned."""
422
423     match = diffTreePattern().next().match(entry)
424     if match:
425         return {
426             'src_mode': match.group(1),
427             'dst_mode': match.group(2),
428             'src_sha1': match.group(3),
429             'dst_sha1': match.group(4),
430             'status': match.group(5),
431             'status_score': match.group(6),
432             'src': match.group(7),
433             'dst': match.group(10)
434         }
435     return None
436
437 def isModeExec(mode):
438     # Returns True if the given git mode represents an executable file,
439     # otherwise False.
440     return mode[-3:] == "755"
441
442 def isModeExecChanged(src_mode, dst_mode):
443     return isModeExec(src_mode) != isModeExec(dst_mode)
444
445 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
446
447     if isinstance(cmd,basestring):
448         cmd = "-G " + cmd
449         expand = True
450     else:
451         cmd = ["-G"] + cmd
452         expand = False
453
454     cmd = p4_build_cmd(cmd)
455     if verbose:
456         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
457
458     # Use a temporary file to avoid deadlocks without
459     # subprocess.communicate(), which would put another copy
460     # of stdout into memory.
461     stdin_file = None
462     if stdin is not None:
463         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
464         if isinstance(stdin,basestring):
465             stdin_file.write(stdin)
466         else:
467             for i in stdin:
468                 stdin_file.write(i + '\n')
469         stdin_file.flush()
470         stdin_file.seek(0)
471
472     p4 = subprocess.Popen(cmd,
473                           shell=expand,
474                           stdin=stdin_file,
475                           stdout=subprocess.PIPE)
476
477     result = []
478     try:
479         while True:
480             entry = marshal.load(p4.stdout)
481             if cb is not None:
482                 cb(entry)
483             else:
484                 result.append(entry)
485     except EOFError:
486         pass
487     exitCode = p4.wait()
488     if exitCode != 0:
489         entry = {}
490         entry["p4ExitCode"] = exitCode
491         result.append(entry)
492
493     return result
494
495 def p4Cmd(cmd):
496     list = p4CmdList(cmd)
497     result = {}
498     for entry in list:
499         result.update(entry)
500     return result;
501
502 def p4Where(depotPath):
503     if not depotPath.endswith("/"):
504         depotPath += "/"
505     depotPath = depotPath + "..."
506     outputList = p4CmdList(["where", depotPath])
507     output = None
508     for entry in outputList:
509         if "depotFile" in entry:
510             if entry["depotFile"] == depotPath:
511                 output = entry
512                 break
513         elif "data" in entry:
514             data = entry.get("data")
515             space = data.find(" ")
516             if data[:space] == depotPath:
517                 output = entry
518                 break
519     if output == None:
520         return ""
521     if output["code"] == "error":
522         return ""
523     clientPath = ""
524     if "path" in output:
525         clientPath = output.get("path")
526     elif "data" in output:
527         data = output.get("data")
528         lastSpace = data.rfind(" ")
529         clientPath = data[lastSpace + 1:]
530
531     if clientPath.endswith("..."):
532         clientPath = clientPath[:-3]
533     return clientPath
534
535 def currentGitBranch():
536     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
537
538 def isValidGitDir(path):
539     if (os.path.exists(path + "/HEAD")
540         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
541         return True;
542     return False
543
544 def parseRevision(ref):
545     return read_pipe("git rev-parse %s" % ref).strip()
546
547 def branchExists(ref):
548     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
549                      ignore_error=True)
550     return len(rev) > 0
551
552 def extractLogMessageFromGitCommit(commit):
553     logMessage = ""
554
555     ## fixme: title is first line of commit, not 1st paragraph.
556     foundTitle = False
557     for log in read_pipe_lines("git cat-file commit %s" % commit):
558        if not foundTitle:
559            if len(log) == 1:
560                foundTitle = True
561            continue
562
563        logMessage += log
564     return logMessage
565
566 def extractSettingsGitLog(log):
567     values = {}
568     for line in log.split("\n"):
569         line = line.strip()
570         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
571         if not m:
572             continue
573
574         assignments = m.group(1).split (':')
575         for a in assignments:
576             vals = a.split ('=')
577             key = vals[0].strip()
578             val = ('='.join (vals[1:])).strip()
579             if val.endswith ('\"') and val.startswith('"'):
580                 val = val[1:-1]
581
582             values[key] = val
583
584     paths = values.get("depot-paths")
585     if not paths:
586         paths = values.get("depot-path")
587     if paths:
588         values['depot-paths'] = paths.split(',')
589     return values
590
591 def gitBranchExists(branch):
592     proc = subprocess.Popen(["git", "rev-parse", branch],
593                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
594     return proc.wait() == 0;
595
596 _gitConfig = {}
597
598 def gitConfig(key):
599     if not _gitConfig.has_key(key):
600         cmd = [ "git", "config", key ]
601         s = read_pipe(cmd, ignore_error=True)
602         _gitConfig[key] = s.strip()
603     return _gitConfig[key]
604
605 def gitConfigBool(key):
606     """Return a bool, using git config --bool.  It is True only if the
607        variable is set to true, and False if set to false or not present
608        in the config."""
609
610     if not _gitConfig.has_key(key):
611         cmd = [ "git", "config", "--bool", key ]
612         s = read_pipe(cmd, ignore_error=True)
613         v = s.strip()
614         _gitConfig[key] = v == "true"
615     return _gitConfig[key]
616
617 def gitConfigList(key):
618     if not _gitConfig.has_key(key):
619         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
620         _gitConfig[key] = s.strip().split(os.linesep)
621     return _gitConfig[key]
622
623 def p4BranchesInGit(branchesAreInRemotes=True):
624     """Find all the branches whose names start with "p4/", looking
625        in remotes or heads as specified by the argument.  Return
626        a dictionary of { branch: revision } for each one found.
627        The branch names are the short names, without any
628        "p4/" prefix."""
629
630     branches = {}
631
632     cmdline = "git rev-parse --symbolic "
633     if branchesAreInRemotes:
634         cmdline += "--remotes"
635     else:
636         cmdline += "--branches"
637
638     for line in read_pipe_lines(cmdline):
639         line = line.strip()
640
641         # only import to p4/
642         if not line.startswith('p4/'):
643             continue
644         # special symbolic ref to p4/master
645         if line == "p4/HEAD":
646             continue
647
648         # strip off p4/ prefix
649         branch = line[len("p4/"):]
650
651         branches[branch] = parseRevision(line)
652
653     return branches
654
655 def branch_exists(branch):
656     """Make sure that the given ref name really exists."""
657
658     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
659     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
660     out, _ = p.communicate()
661     if p.returncode:
662         return False
663     # expect exactly one line of output: the branch name
664     return out.rstrip() == branch
665
666 def findUpstreamBranchPoint(head = "HEAD"):
667     branches = p4BranchesInGit()
668     # map from depot-path to branch name
669     branchByDepotPath = {}
670     for branch in branches.keys():
671         tip = branches[branch]
672         log = extractLogMessageFromGitCommit(tip)
673         settings = extractSettingsGitLog(log)
674         if settings.has_key("depot-paths"):
675             paths = ",".join(settings["depot-paths"])
676             branchByDepotPath[paths] = "remotes/p4/" + branch
677
678     settings = None
679     parent = 0
680     while parent < 65535:
681         commit = head + "~%s" % parent
682         log = extractLogMessageFromGitCommit(commit)
683         settings = extractSettingsGitLog(log)
684         if settings.has_key("depot-paths"):
685             paths = ",".join(settings["depot-paths"])
686             if branchByDepotPath.has_key(paths):
687                 return [branchByDepotPath[paths], settings]
688
689         parent = parent + 1
690
691     return ["", settings]
692
693 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
694     if not silent:
695         print ("Creating/updating branch(es) in %s based on origin branch(es)"
696                % localRefPrefix)
697
698     originPrefix = "origin/p4/"
699
700     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
701         line = line.strip()
702         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
703             continue
704
705         headName = line[len(originPrefix):]
706         remoteHead = localRefPrefix + headName
707         originHead = line
708
709         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
710         if (not original.has_key('depot-paths')
711             or not original.has_key('change')):
712             continue
713
714         update = False
715         if not gitBranchExists(remoteHead):
716             if verbose:
717                 print "creating %s" % remoteHead
718             update = True
719         else:
720             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
721             if settings.has_key('change') > 0:
722                 if settings['depot-paths'] == original['depot-paths']:
723                     originP4Change = int(original['change'])
724                     p4Change = int(settings['change'])
725                     if originP4Change > p4Change:
726                         print ("%s (%s) is newer than %s (%s). "
727                                "Updating p4 branch from origin."
728                                % (originHead, originP4Change,
729                                   remoteHead, p4Change))
730                         update = True
731                 else:
732                     print ("Ignoring: %s was imported from %s while "
733                            "%s was imported from %s"
734                            % (originHead, ','.join(original['depot-paths']),
735                               remoteHead, ','.join(settings['depot-paths'])))
736
737         if update:
738             system("git update-ref %s %s" % (remoteHead, originHead))
739
740 def originP4BranchesExist():
741         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
742
743 def p4ChangesForPaths(depotPaths, changeRange):
744     assert depotPaths
745     cmd = ['changes']
746     for p in depotPaths:
747         cmd += ["%s...%s" % (p, changeRange)]
748     output = p4_read_pipe_lines(cmd)
749
750     changes = {}
751     for line in output:
752         changeNum = int(line.split(" ")[1])
753         changes[changeNum] = True
754
755     changelist = changes.keys()
756     changelist.sort()
757     return changelist
758
759 def p4PathStartsWith(path, prefix):
760     # This method tries to remedy a potential mixed-case issue:
761     #
762     # If UserA adds  //depot/DirA/file1
763     # and UserB adds //depot/dira/file2
764     #
765     # we may or may not have a problem. If you have core.ignorecase=true,
766     # we treat DirA and dira as the same directory
767     if gitConfigBool("core.ignorecase"):
768         return path.lower().startswith(prefix.lower())
769     return path.startswith(prefix)
770
771 def getClientSpec():
772     """Look at the p4 client spec, create a View() object that contains
773        all the mappings, and return it."""
774
775     specList = p4CmdList("client -o")
776     if len(specList) != 1:
777         die('Output from "client -o" is %d lines, expecting 1' %
778             len(specList))
779
780     # dictionary of all client parameters
781     entry = specList[0]
782
783     # just the keys that start with "View"
784     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
785
786     # hold this new View
787     view = View()
788
789     # append the lines, in order, to the view
790     for view_num in range(len(view_keys)):
791         k = "View%d" % view_num
792         if k not in view_keys:
793             die("Expected view key %s missing" % k)
794         view.append(entry[k])
795
796     return view
797
798 def getClientRoot():
799     """Grab the client directory."""
800
801     output = p4CmdList("client -o")
802     if len(output) != 1:
803         die('Output from "client -o" is %d lines, expecting 1' % len(output))
804
805     entry = output[0]
806     if "Root" not in entry:
807         die('Client has no "Root"')
808
809     return entry["Root"]
810
811 #
812 # P4 wildcards are not allowed in filenames.  P4 complains
813 # if you simply add them, but you can force it with "-f", in
814 # which case it translates them into %xx encoding internally.
815 #
816 def wildcard_decode(path):
817     # Search for and fix just these four characters.  Do % last so
818     # that fixing it does not inadvertently create new %-escapes.
819     # Cannot have * in a filename in windows; untested as to
820     # what p4 would do in such a case.
821     if not platform.system() == "Windows":
822         path = path.replace("%2A", "*")
823     path = path.replace("%23", "#") \
824                .replace("%40", "@") \
825                .replace("%25", "%")
826     return path
827
828 def wildcard_encode(path):
829     # do % first to avoid double-encoding the %s introduced here
830     path = path.replace("%", "%25") \
831                .replace("*", "%2A") \
832                .replace("#", "%23") \
833                .replace("@", "%40")
834     return path
835
836 def wildcard_present(path):
837     m = re.search("[*#@%]", path)
838     return m is not None
839
840 class Command:
841     def __init__(self):
842         self.usage = "usage: %prog [options]"
843         self.needsGit = True
844         self.verbose = False
845
846 class P4UserMap:
847     def __init__(self):
848         self.userMapFromPerforceServer = False
849         self.myP4UserId = None
850
851     def p4UserId(self):
852         if self.myP4UserId:
853             return self.myP4UserId
854
855         results = p4CmdList("user -o")
856         for r in results:
857             if r.has_key('User'):
858                 self.myP4UserId = r['User']
859                 return r['User']
860         die("Could not find your p4 user id")
861
862     def p4UserIsMe(self, p4User):
863         # return True if the given p4 user is actually me
864         me = self.p4UserId()
865         if not p4User or p4User != me:
866             return False
867         else:
868             return True
869
870     def getUserCacheFilename(self):
871         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
872         return home + "/.gitp4-usercache.txt"
873
874     def getUserMapFromPerforceServer(self):
875         if self.userMapFromPerforceServer:
876             return
877         self.users = {}
878         self.emails = {}
879
880         for output in p4CmdList("users"):
881             if not output.has_key("User"):
882                 continue
883             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
884             self.emails[output["Email"]] = output["User"]
885
886
887         s = ''
888         for (key, val) in self.users.items():
889             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
890
891         open(self.getUserCacheFilename(), "wb").write(s)
892         self.userMapFromPerforceServer = True
893
894     def loadUserMapFromCache(self):
895         self.users = {}
896         self.userMapFromPerforceServer = False
897         try:
898             cache = open(self.getUserCacheFilename(), "rb")
899             lines = cache.readlines()
900             cache.close()
901             for line in lines:
902                 entry = line.strip().split("\t")
903                 self.users[entry[0]] = entry[1]
904         except IOError:
905             self.getUserMapFromPerforceServer()
906
907 class P4Debug(Command):
908     def __init__(self):
909         Command.__init__(self)
910         self.options = []
911         self.description = "A tool to debug the output of p4 -G."
912         self.needsGit = False
913
914     def run(self, args):
915         j = 0
916         for output in p4CmdList(args):
917             print 'Element: %d' % j
918             j += 1
919             print output
920         return True
921
922 class P4RollBack(Command):
923     def __init__(self):
924         Command.__init__(self)
925         self.options = [
926             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
927         ]
928         self.description = "A tool to debug the multi-branch import. Don't use :)"
929         self.rollbackLocalBranches = False
930
931     def run(self, args):
932         if len(args) != 1:
933             return False
934         maxChange = int(args[0])
935
936         if "p4ExitCode" in p4Cmd("changes -m 1"):
937             die("Problems executing p4");
938
939         if self.rollbackLocalBranches:
940             refPrefix = "refs/heads/"
941             lines = read_pipe_lines("git rev-parse --symbolic --branches")
942         else:
943             refPrefix = "refs/remotes/"
944             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
945
946         for line in lines:
947             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
948                 line = line.strip()
949                 ref = refPrefix + line
950                 log = extractLogMessageFromGitCommit(ref)
951                 settings = extractSettingsGitLog(log)
952
953                 depotPaths = settings['depot-paths']
954                 change = settings['change']
955
956                 changed = False
957
958                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
959                                                            for p in depotPaths]))) == 0:
960                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
961                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
962                     continue
963
964                 while change and int(change) > maxChange:
965                     changed = True
966                     if self.verbose:
967                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
968                     system("git update-ref %s \"%s^\"" % (ref, ref))
969                     log = extractLogMessageFromGitCommit(ref)
970                     settings =  extractSettingsGitLog(log)
971
972
973                     depotPaths = settings['depot-paths']
974                     change = settings['change']
975
976                 if changed:
977                     print "%s rewound to %s" % (ref, change)
978
979         return True
980
981 class P4Submit(Command, P4UserMap):
982
983     conflict_behavior_choices = ("ask", "skip", "quit")
984
985     def __init__(self):
986         Command.__init__(self)
987         P4UserMap.__init__(self)
988         self.options = [
989                 optparse.make_option("--origin", dest="origin"),
990                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
991                 # preserve the user, requires relevant p4 permissions
992                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
993                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
994                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
995                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
996                 optparse.make_option("--conflict", dest="conflict_behavior",
997                                      choices=self.conflict_behavior_choices),
998                 optparse.make_option("--branch", dest="branch"),
999         ]
1000         self.description = "Submit changes from git to the perforce depot."
1001         self.usage += " [name of git branch to submit into perforce depot]"
1002         self.origin = ""
1003         self.detectRenames = False
1004         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1005         self.dry_run = False
1006         self.prepare_p4_only = False
1007         self.conflict_behavior = None
1008         self.isWindows = (platform.system() == "Windows")
1009         self.exportLabels = False
1010         self.p4HasMoveCommand = p4_has_move_command()
1011         self.branch = None
1012
1013     def check(self):
1014         if len(p4CmdList("opened ...")) > 0:
1015             die("You have files opened with perforce! Close them before starting the sync.")
1016
1017     def separate_jobs_from_description(self, message):
1018         """Extract and return a possible Jobs field in the commit
1019            message.  It goes into a separate section in the p4 change
1020            specification.
1021
1022            A jobs line starts with "Jobs:" and looks like a new field
1023            in a form.  Values are white-space separated on the same
1024            line or on following lines that start with a tab.
1025
1026            This does not parse and extract the full git commit message
1027            like a p4 form.  It just sees the Jobs: line as a marker
1028            to pass everything from then on directly into the p4 form,
1029            but outside the description section.
1030
1031            Return a tuple (stripped log message, jobs string)."""
1032
1033         m = re.search(r'^Jobs:', message, re.MULTILINE)
1034         if m is None:
1035             return (message, None)
1036
1037         jobtext = message[m.start():]
1038         stripped_message = message[:m.start()].rstrip()
1039         return (stripped_message, jobtext)
1040
1041     def prepareLogMessage(self, template, message, jobs):
1042         """Edits the template returned from "p4 change -o" to insert
1043            the message in the Description field, and the jobs text in
1044            the Jobs field."""
1045         result = ""
1046
1047         inDescriptionSection = False
1048
1049         for line in template.split("\n"):
1050             if line.startswith("#"):
1051                 result += line + "\n"
1052                 continue
1053
1054             if inDescriptionSection:
1055                 if line.startswith("Files:") or line.startswith("Jobs:"):
1056                     inDescriptionSection = False
1057                     # insert Jobs section
1058                     if jobs:
1059                         result += jobs + "\n"
1060                 else:
1061                     continue
1062             else:
1063                 if line.startswith("Description:"):
1064                     inDescriptionSection = True
1065                     line += "\n"
1066                     for messageLine in message.split("\n"):
1067                         line += "\t" + messageLine + "\n"
1068
1069             result += line + "\n"
1070
1071         return result
1072
1073     def patchRCSKeywords(self, file, pattern):
1074         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1075         (handle, outFileName) = tempfile.mkstemp(dir='.')
1076         try:
1077             outFile = os.fdopen(handle, "w+")
1078             inFile = open(file, "r")
1079             regexp = re.compile(pattern, re.VERBOSE)
1080             for line in inFile.readlines():
1081                 line = regexp.sub(r'$\1$', line)
1082                 outFile.write(line)
1083             inFile.close()
1084             outFile.close()
1085             # Forcibly overwrite the original file
1086             os.unlink(file)
1087             shutil.move(outFileName, file)
1088         except:
1089             # cleanup our temporary file
1090             os.unlink(outFileName)
1091             print "Failed to strip RCS keywords in %s" % file
1092             raise
1093
1094         print "Patched up RCS keywords in %s" % file
1095
1096     def p4UserForCommit(self,id):
1097         # Return the tuple (perforce user,git email) for a given git commit id
1098         self.getUserMapFromPerforceServer()
1099         gitEmail = read_pipe(["git", "log", "--max-count=1",
1100                               "--format=%ae", id])
1101         gitEmail = gitEmail.strip()
1102         if not self.emails.has_key(gitEmail):
1103             return (None,gitEmail)
1104         else:
1105             return (self.emails[gitEmail],gitEmail)
1106
1107     def checkValidP4Users(self,commits):
1108         # check if any git authors cannot be mapped to p4 users
1109         for id in commits:
1110             (user,email) = self.p4UserForCommit(id)
1111             if not user:
1112                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1113                 if gitConfigBool("git-p4.allowMissingP4Users"):
1114                     print "%s" % msg
1115                 else:
1116                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1117
1118     def lastP4Changelist(self):
1119         # Get back the last changelist number submitted in this client spec. This
1120         # then gets used to patch up the username in the change. If the same
1121         # client spec is being used by multiple processes then this might go
1122         # wrong.
1123         results = p4CmdList("client -o")        # find the current client
1124         client = None
1125         for r in results:
1126             if r.has_key('Client'):
1127                 client = r['Client']
1128                 break
1129         if not client:
1130             die("could not get client spec")
1131         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1132         for r in results:
1133             if r.has_key('change'):
1134                 return r['change']
1135         die("Could not get changelist number for last submit - cannot patch up user details")
1136
1137     def modifyChangelistUser(self, changelist, newUser):
1138         # fixup the user field of a changelist after it has been submitted.
1139         changes = p4CmdList("change -o %s" % changelist)
1140         if len(changes) != 1:
1141             die("Bad output from p4 change modifying %s to user %s" %
1142                 (changelist, newUser))
1143
1144         c = changes[0]
1145         if c['User'] == newUser: return   # nothing to do
1146         c['User'] = newUser
1147         input = marshal.dumps(c)
1148
1149         result = p4CmdList("change -f -i", stdin=input)
1150         for r in result:
1151             if r.has_key('code'):
1152                 if r['code'] == 'error':
1153                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1154             if r.has_key('data'):
1155                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1156                 return
1157         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1158
1159     def canChangeChangelists(self):
1160         # check to see if we have p4 admin or super-user permissions, either of
1161         # which are required to modify changelists.
1162         results = p4CmdList(["protects", self.depotPath])
1163         for r in results:
1164             if r.has_key('perm'):
1165                 if r['perm'] == 'admin':
1166                     return 1
1167                 if r['perm'] == 'super':
1168                     return 1
1169         return 0
1170
1171     def prepareSubmitTemplate(self):
1172         """Run "p4 change -o" to grab a change specification template.
1173            This does not use "p4 -G", as it is nice to keep the submission
1174            template in original order, since a human might edit it.
1175
1176            Remove lines in the Files section that show changes to files
1177            outside the depot path we're committing into."""
1178
1179         template = ""
1180         inFilesSection = False
1181         for line in p4_read_pipe_lines(['change', '-o']):
1182             if line.endswith("\r\n"):
1183                 line = line[:-2] + "\n"
1184             if inFilesSection:
1185                 if line.startswith("\t"):
1186                     # path starts and ends with a tab
1187                     path = line[1:]
1188                     lastTab = path.rfind("\t")
1189                     if lastTab != -1:
1190                         path = path[:lastTab]
1191                         if not p4PathStartsWith(path, self.depotPath):
1192                             continue
1193                 else:
1194                     inFilesSection = False
1195             else:
1196                 if line.startswith("Files:"):
1197                     inFilesSection = True
1198
1199             template += line
1200
1201         return template
1202
1203     def edit_template(self, template_file):
1204         """Invoke the editor to let the user change the submission
1205            message.  Return true if okay to continue with the submit."""
1206
1207         # if configured to skip the editing part, just submit
1208         if gitConfigBool("git-p4.skipSubmitEdit"):
1209             return True
1210
1211         # look at the modification time, to check later if the user saved
1212         # the file
1213         mtime = os.stat(template_file).st_mtime
1214
1215         # invoke the editor
1216         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1217             editor = os.environ.get("P4EDITOR")
1218         else:
1219             editor = read_pipe("git var GIT_EDITOR").strip()
1220         system(editor + " " + template_file)
1221
1222         # If the file was not saved, prompt to see if this patch should
1223         # be skipped.  But skip this verification step if configured so.
1224         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1225             return True
1226
1227         # modification time updated means user saved the file
1228         if os.stat(template_file).st_mtime > mtime:
1229             return True
1230
1231         while True:
1232             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1233             if response == 'y':
1234                 return True
1235             if response == 'n':
1236                 return False
1237
1238     def applyCommit(self, id):
1239         """Apply one commit, return True if it succeeded."""
1240
1241         print "Applying", read_pipe(["git", "show", "-s",
1242                                      "--format=format:%h %s", id])
1243
1244         (p4User, gitEmail) = self.p4UserForCommit(id)
1245
1246         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1247         filesToAdd = set()
1248         filesToDelete = set()
1249         editedFiles = set()
1250         pureRenameCopy = set()
1251         filesToChangeExecBit = {}
1252
1253         for line in diff:
1254             diff = parseDiffTreeEntry(line)
1255             modifier = diff['status']
1256             path = diff['src']
1257             if modifier == "M":
1258                 p4_edit(path)
1259                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1260                     filesToChangeExecBit[path] = diff['dst_mode']
1261                 editedFiles.add(path)
1262             elif modifier == "A":
1263                 filesToAdd.add(path)
1264                 filesToChangeExecBit[path] = diff['dst_mode']
1265                 if path in filesToDelete:
1266                     filesToDelete.remove(path)
1267             elif modifier == "D":
1268                 filesToDelete.add(path)
1269                 if path in filesToAdd:
1270                     filesToAdd.remove(path)
1271             elif modifier == "C":
1272                 src, dest = diff['src'], diff['dst']
1273                 p4_integrate(src, dest)
1274                 pureRenameCopy.add(dest)
1275                 if diff['src_sha1'] != diff['dst_sha1']:
1276                     p4_edit(dest)
1277                     pureRenameCopy.discard(dest)
1278                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1279                     p4_edit(dest)
1280                     pureRenameCopy.discard(dest)
1281                     filesToChangeExecBit[dest] = diff['dst_mode']
1282                 if self.isWindows:
1283                     # turn off read-only attribute
1284                     os.chmod(dest, stat.S_IWRITE)
1285                 os.unlink(dest)
1286                 editedFiles.add(dest)
1287             elif modifier == "R":
1288                 src, dest = diff['src'], diff['dst']
1289                 if self.p4HasMoveCommand:
1290                     p4_edit(src)        # src must be open before move
1291                     p4_move(src, dest)  # opens for (move/delete, move/add)
1292                 else:
1293                     p4_integrate(src, dest)
1294                     if diff['src_sha1'] != diff['dst_sha1']:
1295                         p4_edit(dest)
1296                     else:
1297                         pureRenameCopy.add(dest)
1298                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1299                     if not self.p4HasMoveCommand:
1300                         p4_edit(dest)   # with move: already open, writable
1301                     filesToChangeExecBit[dest] = diff['dst_mode']
1302                 if not self.p4HasMoveCommand:
1303                     if self.isWindows:
1304                         os.chmod(dest, stat.S_IWRITE)
1305                     os.unlink(dest)
1306                     filesToDelete.add(src)
1307                 editedFiles.add(dest)
1308             else:
1309                 die("unknown modifier %s for %s" % (modifier, path))
1310
1311         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1312         patchcmd = diffcmd + " | git apply "
1313         tryPatchCmd = patchcmd + "--check -"
1314         applyPatchCmd = patchcmd + "--check --apply -"
1315         patch_succeeded = True
1316
1317         if os.system(tryPatchCmd) != 0:
1318             fixed_rcs_keywords = False
1319             patch_succeeded = False
1320             print "Unfortunately applying the change failed!"
1321
1322             # Patch failed, maybe it's just RCS keyword woes. Look through
1323             # the patch to see if that's possible.
1324             if gitConfigBool("git-p4.attemptRCSCleanup"):
1325                 file = None
1326                 pattern = None
1327                 kwfiles = {}
1328                 for file in editedFiles | filesToDelete:
1329                     # did this file's delta contain RCS keywords?
1330                     pattern = p4_keywords_regexp_for_file(file)
1331
1332                     if pattern:
1333                         # this file is a possibility...look for RCS keywords.
1334                         regexp = re.compile(pattern, re.VERBOSE)
1335                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1336                             if regexp.search(line):
1337                                 if verbose:
1338                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1339                                 kwfiles[file] = pattern
1340                                 break
1341
1342                 for file in kwfiles:
1343                     if verbose:
1344                         print "zapping %s with %s" % (line,pattern)
1345                     # File is being deleted, so not open in p4.  Must
1346                     # disable the read-only bit on windows.
1347                     if self.isWindows and file not in editedFiles:
1348                         os.chmod(file, stat.S_IWRITE)
1349                     self.patchRCSKeywords(file, kwfiles[file])
1350                     fixed_rcs_keywords = True
1351
1352             if fixed_rcs_keywords:
1353                 print "Retrying the patch with RCS keywords cleaned up"
1354                 if os.system(tryPatchCmd) == 0:
1355                     patch_succeeded = True
1356
1357         if not patch_succeeded:
1358             for f in editedFiles:
1359                 p4_revert(f)
1360             return False
1361
1362         #
1363         # Apply the patch for real, and do add/delete/+x handling.
1364         #
1365         system(applyPatchCmd)
1366
1367         for f in filesToAdd:
1368             p4_add(f)
1369         for f in filesToDelete:
1370             p4_revert(f)
1371             p4_delete(f)
1372
1373         # Set/clear executable bits
1374         for f in filesToChangeExecBit.keys():
1375             mode = filesToChangeExecBit[f]
1376             setP4ExecBit(f, mode)
1377
1378         #
1379         # Build p4 change description, starting with the contents
1380         # of the git commit message.
1381         #
1382         logMessage = extractLogMessageFromGitCommit(id)
1383         logMessage = logMessage.strip()
1384         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1385
1386         template = self.prepareSubmitTemplate()
1387         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1388
1389         if self.preserveUser:
1390            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1391
1392         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1393             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1394             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1395             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1396
1397         separatorLine = "######## everything below this line is just the diff #######\n"
1398
1399         # diff
1400         if os.environ.has_key("P4DIFF"):
1401             del(os.environ["P4DIFF"])
1402         diff = ""
1403         for editedFile in editedFiles:
1404             diff += p4_read_pipe(['diff', '-du',
1405                                   wildcard_encode(editedFile)])
1406
1407         # new file diff
1408         newdiff = ""
1409         for newFile in filesToAdd:
1410             newdiff += "==== new file ====\n"
1411             newdiff += "--- /dev/null\n"
1412             newdiff += "+++ %s\n" % newFile
1413             f = open(newFile, "r")
1414             for line in f.readlines():
1415                 newdiff += "+" + line
1416             f.close()
1417
1418         # change description file: submitTemplate, separatorLine, diff, newdiff
1419         (handle, fileName) = tempfile.mkstemp()
1420         tmpFile = os.fdopen(handle, "w+")
1421         if self.isWindows:
1422             submitTemplate = submitTemplate.replace("\n", "\r\n")
1423             separatorLine = separatorLine.replace("\n", "\r\n")
1424             newdiff = newdiff.replace("\n", "\r\n")
1425         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1426         tmpFile.close()
1427
1428         if self.prepare_p4_only:
1429             #
1430             # Leave the p4 tree prepared, and the submit template around
1431             # and let the user decide what to do next
1432             #
1433             print
1434             print "P4 workspace prepared for submission."
1435             print "To submit or revert, go to client workspace"
1436             print "  " + self.clientPath
1437             print
1438             print "To submit, use \"p4 submit\" to write a new description,"
1439             print "or \"p4 submit -i %s\" to use the one prepared by" \
1440                   " \"git p4\"." % fileName
1441             print "You can delete the file \"%s\" when finished." % fileName
1442
1443             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1444                 print "To preserve change ownership by user %s, you must\n" \
1445                       "do \"p4 change -f <change>\" after submitting and\n" \
1446                       "edit the User field."
1447             if pureRenameCopy:
1448                 print "After submitting, renamed files must be re-synced."
1449                 print "Invoke \"p4 sync -f\" on each of these files:"
1450                 for f in pureRenameCopy:
1451                     print "  " + f
1452
1453             print
1454             print "To revert the changes, use \"p4 revert ...\", and delete"
1455             print "the submit template file \"%s\"" % fileName
1456             if filesToAdd:
1457                 print "Since the commit adds new files, they must be deleted:"
1458                 for f in filesToAdd:
1459                     print "  " + f
1460             print
1461             return True
1462
1463         #
1464         # Let the user edit the change description, then submit it.
1465         #
1466         if self.edit_template(fileName):
1467             # read the edited message and submit
1468             ret = True
1469             tmpFile = open(fileName, "rb")
1470             message = tmpFile.read()
1471             tmpFile.close()
1472             submitTemplate = message[:message.index(separatorLine)]
1473             if self.isWindows:
1474                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1475             p4_write_pipe(['submit', '-i'], submitTemplate)
1476
1477             if self.preserveUser:
1478                 if p4User:
1479                     # Get last changelist number. Cannot easily get it from
1480                     # the submit command output as the output is
1481                     # unmarshalled.
1482                     changelist = self.lastP4Changelist()
1483                     self.modifyChangelistUser(changelist, p4User)
1484
1485             # The rename/copy happened by applying a patch that created a
1486             # new file.  This leaves it writable, which confuses p4.
1487             for f in pureRenameCopy:
1488                 p4_sync(f, "-f")
1489
1490         else:
1491             # skip this patch
1492             ret = False
1493             print "Submission cancelled, undoing p4 changes."
1494             for f in editedFiles:
1495                 p4_revert(f)
1496             for f in filesToAdd:
1497                 p4_revert(f)
1498                 os.remove(f)
1499             for f in filesToDelete:
1500                 p4_revert(f)
1501
1502         os.remove(fileName)
1503         return ret
1504
1505     # Export git tags as p4 labels. Create a p4 label and then tag
1506     # with that.
1507     def exportGitTags(self, gitTags):
1508         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1509         if len(validLabelRegexp) == 0:
1510             validLabelRegexp = defaultLabelRegexp
1511         m = re.compile(validLabelRegexp)
1512
1513         for name in gitTags:
1514
1515             if not m.match(name):
1516                 if verbose:
1517                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1518                 continue
1519
1520             # Get the p4 commit this corresponds to
1521             logMessage = extractLogMessageFromGitCommit(name)
1522             values = extractSettingsGitLog(logMessage)
1523
1524             if not values.has_key('change'):
1525                 # a tag pointing to something not sent to p4; ignore
1526                 if verbose:
1527                     print "git tag %s does not give a p4 commit" % name
1528                 continue
1529             else:
1530                 changelist = values['change']
1531
1532             # Get the tag details.
1533             inHeader = True
1534             isAnnotated = False
1535             body = []
1536             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1537                 l = l.strip()
1538                 if inHeader:
1539                     if re.match(r'tag\s+', l):
1540                         isAnnotated = True
1541                     elif re.match(r'\s*$', l):
1542                         inHeader = False
1543                         continue
1544                 else:
1545                     body.append(l)
1546
1547             if not isAnnotated:
1548                 body = ["lightweight tag imported by git p4\n"]
1549
1550             # Create the label - use the same view as the client spec we are using
1551             clientSpec = getClientSpec()
1552
1553             labelTemplate  = "Label: %s\n" % name
1554             labelTemplate += "Description:\n"
1555             for b in body:
1556                 labelTemplate += "\t" + b + "\n"
1557             labelTemplate += "View:\n"
1558             for mapping in clientSpec.mappings:
1559                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1560
1561             if self.dry_run:
1562                 print "Would create p4 label %s for tag" % name
1563             elif self.prepare_p4_only:
1564                 print "Not creating p4 label %s for tag due to option" \
1565                       " --prepare-p4-only" % name
1566             else:
1567                 p4_write_pipe(["label", "-i"], labelTemplate)
1568
1569                 # Use the label
1570                 p4_system(["tag", "-l", name] +
1571                           ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1572
1573                 if verbose:
1574                     print "created p4 label for tag %s" % name
1575
1576     def run(self, args):
1577         if len(args) == 0:
1578             self.master = currentGitBranch()
1579             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1580                 die("Detecting current git branch failed!")
1581         elif len(args) == 1:
1582             self.master = args[0]
1583             if not branchExists(self.master):
1584                 die("Branch %s does not exist" % self.master)
1585         else:
1586             return False
1587
1588         allowSubmit = gitConfig("git-p4.allowSubmit")
1589         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1590             die("%s is not in git-p4.allowSubmit" % self.master)
1591
1592         [upstream, settings] = findUpstreamBranchPoint()
1593         self.depotPath = settings['depot-paths'][0]
1594         if len(self.origin) == 0:
1595             self.origin = upstream
1596
1597         if self.preserveUser:
1598             if not self.canChangeChangelists():
1599                 die("Cannot preserve user names without p4 super-user or admin permissions")
1600
1601         # if not set from the command line, try the config file
1602         if self.conflict_behavior is None:
1603             val = gitConfig("git-p4.conflict")
1604             if val:
1605                 if val not in self.conflict_behavior_choices:
1606                     die("Invalid value '%s' for config git-p4.conflict" % val)
1607             else:
1608                 val = "ask"
1609             self.conflict_behavior = val
1610
1611         if self.verbose:
1612             print "Origin branch is " + self.origin
1613
1614         if len(self.depotPath) == 0:
1615             print "Internal error: cannot locate perforce depot path from existing branches"
1616             sys.exit(128)
1617
1618         self.useClientSpec = False
1619         if gitConfigBool("git-p4.useclientspec"):
1620             self.useClientSpec = True
1621         if self.useClientSpec:
1622             self.clientSpecDirs = getClientSpec()
1623
1624         if self.useClientSpec:
1625             # all files are relative to the client spec
1626             self.clientPath = getClientRoot()
1627         else:
1628             self.clientPath = p4Where(self.depotPath)
1629
1630         if self.clientPath == "":
1631             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1632
1633         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1634         self.oldWorkingDirectory = os.getcwd()
1635
1636         # ensure the clientPath exists
1637         new_client_dir = False
1638         if not os.path.exists(self.clientPath):
1639             new_client_dir = True
1640             os.makedirs(self.clientPath)
1641
1642         chdir(self.clientPath, is_client_path=True)
1643         if self.dry_run:
1644             print "Would synchronize p4 checkout in %s" % self.clientPath
1645         else:
1646             print "Synchronizing p4 checkout..."
1647             if new_client_dir:
1648                 # old one was destroyed, and maybe nobody told p4
1649                 p4_sync("...", "-f")
1650             else:
1651                 p4_sync("...")
1652         self.check()
1653
1654         commits = []
1655         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1656             commits.append(line.strip())
1657         commits.reverse()
1658
1659         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1660             self.checkAuthorship = False
1661         else:
1662             self.checkAuthorship = True
1663
1664         if self.preserveUser:
1665             self.checkValidP4Users(commits)
1666
1667         #
1668         # Build up a set of options to be passed to diff when
1669         # submitting each commit to p4.
1670         #
1671         if self.detectRenames:
1672             # command-line -M arg
1673             self.diffOpts = "-M"
1674         else:
1675             # If not explicitly set check the config variable
1676             detectRenames = gitConfig("git-p4.detectRenames")
1677
1678             if detectRenames.lower() == "false" or detectRenames == "":
1679                 self.diffOpts = ""
1680             elif detectRenames.lower() == "true":
1681                 self.diffOpts = "-M"
1682             else:
1683                 self.diffOpts = "-M%s" % detectRenames
1684
1685         # no command-line arg for -C or --find-copies-harder, just
1686         # config variables
1687         detectCopies = gitConfig("git-p4.detectCopies")
1688         if detectCopies.lower() == "false" or detectCopies == "":
1689             pass
1690         elif detectCopies.lower() == "true":
1691             self.diffOpts += " -C"
1692         else:
1693             self.diffOpts += " -C%s" % detectCopies
1694
1695         if gitConfigBool("git-p4.detectCopiesHarder"):
1696             self.diffOpts += " --find-copies-harder"
1697
1698         #
1699         # Apply the commits, one at a time.  On failure, ask if should
1700         # continue to try the rest of the patches, or quit.
1701         #
1702         if self.dry_run:
1703             print "Would apply"
1704         applied = []
1705         last = len(commits) - 1
1706         for i, commit in enumerate(commits):
1707             if self.dry_run:
1708                 print " ", read_pipe(["git", "show", "-s",
1709                                       "--format=format:%h %s", commit])
1710                 ok = True
1711             else:
1712                 ok = self.applyCommit(commit)
1713             if ok:
1714                 applied.append(commit)
1715             else:
1716                 if self.prepare_p4_only and i < last:
1717                     print "Processing only the first commit due to option" \
1718                           " --prepare-p4-only"
1719                     break
1720                 if i < last:
1721                     quit = False
1722                     while True:
1723                         # prompt for what to do, or use the option/variable
1724                         if self.conflict_behavior == "ask":
1725                             print "What do you want to do?"
1726                             response = raw_input("[s]kip this commit but apply"
1727                                                  " the rest, or [q]uit? ")
1728                             if not response:
1729                                 continue
1730                         elif self.conflict_behavior == "skip":
1731                             response = "s"
1732                         elif self.conflict_behavior == "quit":
1733                             response = "q"
1734                         else:
1735                             die("Unknown conflict_behavior '%s'" %
1736                                 self.conflict_behavior)
1737
1738                         if response[0] == "s":
1739                             print "Skipping this commit, but applying the rest"
1740                             break
1741                         if response[0] == "q":
1742                             print "Quitting"
1743                             quit = True
1744                             break
1745                     if quit:
1746                         break
1747
1748         chdir(self.oldWorkingDirectory)
1749
1750         if self.dry_run:
1751             pass
1752         elif self.prepare_p4_only:
1753             pass
1754         elif len(commits) == len(applied):
1755             print "All commits applied!"
1756
1757             sync = P4Sync()
1758             if self.branch:
1759                 sync.branch = self.branch
1760             sync.run([])
1761
1762             rebase = P4Rebase()
1763             rebase.rebase()
1764
1765         else:
1766             if len(applied) == 0:
1767                 print "No commits applied."
1768             else:
1769                 print "Applied only the commits marked with '*':"
1770                 for c in commits:
1771                     if c in applied:
1772                         star = "*"
1773                     else:
1774                         star = " "
1775                     print star, read_pipe(["git", "show", "-s",
1776                                            "--format=format:%h %s",  c])
1777                 print "You will have to do 'git p4 sync' and rebase."
1778
1779         if gitConfigBool("git-p4.exportLabels"):
1780             self.exportLabels = True
1781
1782         if self.exportLabels:
1783             p4Labels = getP4Labels(self.depotPath)
1784             gitTags = getGitTags()
1785
1786             missingGitTags = gitTags - p4Labels
1787             self.exportGitTags(missingGitTags)
1788
1789         # exit with error unless everything applied perfectly
1790         if len(commits) != len(applied):
1791                 sys.exit(1)
1792
1793         return True
1794
1795 class View(object):
1796     """Represent a p4 view ("p4 help views"), and map files in a
1797        repo according to the view."""
1798
1799     class Path(object):
1800         """A depot or client path, possibly containing wildcards.
1801            The only one supported is ... at the end, currently.
1802            Initialize with the full path, with //depot or //client."""
1803
1804         def __init__(self, path, is_depot):
1805             self.path = path
1806             self.is_depot = is_depot
1807             self.find_wildcards()
1808             # remember the prefix bit, useful for relative mappings
1809             m = re.match("(//[^/]+/)", self.path)
1810             if not m:
1811                 die("Path %s does not start with //prefix/" % self.path)
1812             prefix = m.group(1)
1813             if not self.is_depot:
1814                 # strip //client/ on client paths
1815                 self.path = self.path[len(prefix):]
1816
1817         def find_wildcards(self):
1818             """Make sure wildcards are valid, and set up internal
1819                variables."""
1820
1821             self.ends_triple_dot = False
1822             # There are three wildcards allowed in p4 views
1823             # (see "p4 help views").  This code knows how to
1824             # handle "..." (only at the end), but cannot deal with
1825             # "%%n" or "*".  Only check the depot_side, as p4 should
1826             # validate that the client_side matches too.
1827             if re.search(r'%%[1-9]', self.path):
1828                 die("Can't handle %%n wildcards in view: %s" % self.path)
1829             if self.path.find("*") >= 0:
1830                 die("Can't handle * wildcards in view: %s" % self.path)
1831             triple_dot_index = self.path.find("...")
1832             if triple_dot_index >= 0:
1833                 if triple_dot_index != len(self.path) - 3:
1834                     die("Can handle only single ... wildcard, at end: %s" %
1835                         self.path)
1836                 self.ends_triple_dot = True
1837
1838         def ensure_compatible(self, other_path):
1839             """Make sure the wildcards agree."""
1840             if self.ends_triple_dot != other_path.ends_triple_dot:
1841                  die("Both paths must end with ... if either does;\n" +
1842                      "paths: %s %s" % (self.path, other_path.path))
1843
1844         def match_wildcards(self, test_path):
1845             """See if this test_path matches us, and fill in the value
1846                of the wildcards if so.  Returns a tuple of
1847                (True|False, wildcards[]).  For now, only the ... at end
1848                is supported, so at most one wildcard."""
1849             if self.ends_triple_dot:
1850                 dotless = self.path[:-3]
1851                 if test_path.startswith(dotless):
1852                     wildcard = test_path[len(dotless):]
1853                     return (True, [ wildcard ])
1854             else:
1855                 if test_path == self.path:
1856                     return (True, [])
1857             return (False, [])
1858
1859         def match(self, test_path):
1860             """Just return if it matches; don't bother with the wildcards."""
1861             b, _ = self.match_wildcards(test_path)
1862             return b
1863
1864         def fill_in_wildcards(self, wildcards):
1865             """Return the relative path, with the wildcards filled in
1866                if there are any."""
1867             if self.ends_triple_dot:
1868                 return self.path[:-3] + wildcards[0]
1869             else:
1870                 return self.path
1871
1872     class Mapping(object):
1873         def __init__(self, depot_side, client_side, overlay, exclude):
1874             # depot_side is without the trailing /... if it had one
1875             self.depot_side = View.Path(depot_side, is_depot=True)
1876             self.client_side = View.Path(client_side, is_depot=False)
1877             self.overlay = overlay  # started with "+"
1878             self.exclude = exclude  # started with "-"
1879             assert not (self.overlay and self.exclude)
1880             self.depot_side.ensure_compatible(self.client_side)
1881
1882         def __str__(self):
1883             c = " "
1884             if self.overlay:
1885                 c = "+"
1886             if self.exclude:
1887                 c = "-"
1888             return "View.Mapping: %s%s -> %s" % \
1889                    (c, self.depot_side.path, self.client_side.path)
1890
1891         def map_depot_to_client(self, depot_path):
1892             """Calculate the client path if using this mapping on the
1893                given depot path; does not consider the effect of other
1894                mappings in a view.  Even excluded mappings are returned."""
1895             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1896             if not matches:
1897                 return ""
1898             client_path = self.client_side.fill_in_wildcards(wildcards)
1899             return client_path
1900
1901     #
1902     # View methods
1903     #
1904     def __init__(self):
1905         self.mappings = []
1906
1907     def append(self, view_line):
1908         """Parse a view line, splitting it into depot and client
1909            sides.  Append to self.mappings, preserving order."""
1910
1911         # Split the view line into exactly two words.  P4 enforces
1912         # structure on these lines that simplifies this quite a bit.
1913         #
1914         # Either or both words may be double-quoted.
1915         # Single quotes do not matter.
1916         # Double-quote marks cannot occur inside the words.
1917         # A + or - prefix is also inside the quotes.
1918         # There are no quotes unless they contain a space.
1919         # The line is already white-space stripped.
1920         # The two words are separated by a single space.
1921         #
1922         if view_line[0] == '"':
1923             # First word is double quoted.  Find its end.
1924             close_quote_index = view_line.find('"', 1)
1925             if close_quote_index <= 0:
1926                 die("No first-word closing quote found: %s" % view_line)
1927             depot_side = view_line[1:close_quote_index]
1928             # skip closing quote and space
1929             rhs_index = close_quote_index + 1 + 1
1930         else:
1931             space_index = view_line.find(" ")
1932             if space_index <= 0:
1933                 die("No word-splitting space found: %s" % view_line)
1934             depot_side = view_line[0:space_index]
1935             rhs_index = space_index + 1
1936
1937         if view_line[rhs_index] == '"':
1938             # Second word is double quoted.  Make sure there is a
1939             # double quote at the end too.
1940             if not view_line.endswith('"'):
1941                 die("View line with rhs quote should end with one: %s" %
1942                     view_line)
1943             # skip the quotes
1944             client_side = view_line[rhs_index+1:-1]
1945         else:
1946             client_side = view_line[rhs_index:]
1947
1948         # prefix + means overlay on previous mapping
1949         overlay = False
1950         if depot_side.startswith("+"):
1951             overlay = True
1952             depot_side = depot_side[1:]
1953
1954         # prefix - means exclude this path
1955         exclude = False
1956         if depot_side.startswith("-"):
1957             exclude = True
1958             depot_side = depot_side[1:]
1959
1960         m = View.Mapping(depot_side, client_side, overlay, exclude)
1961         self.mappings.append(m)
1962
1963     def map_in_client(self, depot_path):
1964         """Return the relative location in the client where this
1965            depot file should live.  Returns "" if the file should
1966            not be mapped in the client."""
1967
1968         paths_filled = []
1969         client_path = ""
1970
1971         # look at later entries first
1972         for m in self.mappings[::-1]:
1973
1974             # see where will this path end up in the client
1975             p = m.map_depot_to_client(depot_path)
1976
1977             if p == "":
1978                 # Depot path does not belong in client.  Must remember
1979                 # this, as previous items should not cause files to
1980                 # exist in this path either.  Remember that the list is
1981                 # being walked from the end, which has higher precedence.
1982                 # Overlap mappings do not exclude previous mappings.
1983                 if not m.overlay:
1984                     paths_filled.append(m.client_side)
1985
1986             else:
1987                 # This mapping matched; no need to search any further.
1988                 # But, the mapping could be rejected if the client path
1989                 # has already been claimed by an earlier mapping (i.e.
1990                 # one later in the list, which we are walking backwards).
1991                 already_mapped_in_client = False
1992                 for f in paths_filled:
1993                     # this is View.Path.match
1994                     if f.match(p):
1995                         already_mapped_in_client = True
1996                         break
1997                 if not already_mapped_in_client:
1998                     # Include this file, unless it is from a line that
1999                     # explicitly said to exclude it.
2000                     if not m.exclude:
2001                         client_path = p
2002
2003                 # a match, even if rejected, always stops the search
2004                 break
2005
2006         return client_path
2007
2008 class P4Sync(Command, P4UserMap):
2009     delete_actions = ( "delete", "move/delete", "purge" )
2010
2011     def __init__(self):
2012         Command.__init__(self)
2013         P4UserMap.__init__(self)
2014         self.options = [
2015                 optparse.make_option("--branch", dest="branch"),
2016                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2017                 optparse.make_option("--changesfile", dest="changesFile"),
2018                 optparse.make_option("--silent", dest="silent", action="store_true"),
2019                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2020                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2021                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2022                                      help="Import into refs/heads/ , not refs/remotes"),
2023                 optparse.make_option("--max-changes", dest="maxChanges"),
2024                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2025                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2026                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2027                                      help="Only sync files that are included in the Perforce Client Spec")
2028         ]
2029         self.description = """Imports from Perforce into a git repository.\n
2030     example:
2031     //depot/my/project/ -- to import the current head
2032     //depot/my/project/@all -- to import everything
2033     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2034
2035     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2036
2037         self.usage += " //depot/path[@revRange]"
2038         self.silent = False
2039         self.createdBranches = set()
2040         self.committedChanges = set()
2041         self.branch = ""
2042         self.detectBranches = False
2043         self.detectLabels = False
2044         self.importLabels = False
2045         self.changesFile = ""
2046         self.syncWithOrigin = True
2047         self.importIntoRemotes = True
2048         self.maxChanges = ""
2049         self.keepRepoPath = False
2050         self.depotPaths = None
2051         self.p4BranchesInGit = []
2052         self.cloneExclude = []
2053         self.useClientSpec = False
2054         self.useClientSpec_from_options = False
2055         self.clientSpecDirs = None
2056         self.tempBranches = []
2057         self.tempBranchLocation = "git-p4-tmp"
2058
2059         if gitConfig("git-p4.syncFromOrigin") == "false":
2060             self.syncWithOrigin = False
2061
2062     # Force a checkpoint in fast-import and wait for it to finish
2063     def checkpoint(self):
2064         self.gitStream.write("checkpoint\n\n")
2065         self.gitStream.write("progress checkpoint\n\n")
2066         out = self.gitOutput.readline()
2067         if self.verbose:
2068             print "checkpoint finished: " + out
2069
2070     def extractFilesFromCommit(self, commit):
2071         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2072                              for path in self.cloneExclude]
2073         files = []
2074         fnum = 0
2075         while commit.has_key("depotFile%s" % fnum):
2076             path =  commit["depotFile%s" % fnum]
2077
2078             if [p for p in self.cloneExclude
2079                 if p4PathStartsWith(path, p)]:
2080                 found = False
2081             else:
2082                 found = [p for p in self.depotPaths
2083                          if p4PathStartsWith(path, p)]
2084             if not found:
2085                 fnum = fnum + 1
2086                 continue
2087
2088             file = {}
2089             file["path"] = path
2090             file["rev"] = commit["rev%s" % fnum]
2091             file["action"] = commit["action%s" % fnum]
2092             file["type"] = commit["type%s" % fnum]
2093             files.append(file)
2094             fnum = fnum + 1
2095         return files
2096
2097     def stripRepoPath(self, path, prefixes):
2098         """When streaming files, this is called to map a p4 depot path
2099            to where it should go in git.  The prefixes are either
2100            self.depotPaths, or self.branchPrefixes in the case of
2101            branch detection."""
2102
2103         if self.useClientSpec:
2104             # branch detection moves files up a level (the branch name)
2105             # from what client spec interpretation gives
2106             path = self.clientSpecDirs.map_in_client(path)
2107             if self.detectBranches:
2108                 for b in self.knownBranches:
2109                     if path.startswith(b + "/"):
2110                         path = path[len(b)+1:]
2111
2112         elif self.keepRepoPath:
2113             # Preserve everything in relative path name except leading
2114             # //depot/; just look at first prefix as they all should
2115             # be in the same depot.
2116             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2117             if p4PathStartsWith(path, depot):
2118                 path = path[len(depot):]
2119
2120         else:
2121             for p in prefixes:
2122                 if p4PathStartsWith(path, p):
2123                     path = path[len(p):]
2124                     break
2125
2126         path = wildcard_decode(path)
2127         return path
2128
2129     def splitFilesIntoBranches(self, commit):
2130         """Look at each depotFile in the commit to figure out to what
2131            branch it belongs."""
2132
2133         branches = {}
2134         fnum = 0
2135         while commit.has_key("depotFile%s" % fnum):
2136             path =  commit["depotFile%s" % fnum]
2137             found = [p for p in self.depotPaths
2138                      if p4PathStartsWith(path, p)]
2139             if not found:
2140                 fnum = fnum + 1
2141                 continue
2142
2143             file = {}
2144             file["path"] = path
2145             file["rev"] = commit["rev%s" % fnum]
2146             file["action"] = commit["action%s" % fnum]
2147             file["type"] = commit["type%s" % fnum]
2148             fnum = fnum + 1
2149
2150             # start with the full relative path where this file would
2151             # go in a p4 client
2152             if self.useClientSpec:
2153                 relPath = self.clientSpecDirs.map_in_client(path)
2154             else:
2155                 relPath = self.stripRepoPath(path, self.depotPaths)
2156
2157             for branch in self.knownBranches.keys():
2158                 # add a trailing slash so that a commit into qt/4.2foo
2159                 # doesn't end up in qt/4.2, e.g.
2160                 if relPath.startswith(branch + "/"):
2161                     if branch not in branches:
2162                         branches[branch] = []
2163                     branches[branch].append(file)
2164                     break
2165
2166         return branches
2167
2168     # output one file from the P4 stream
2169     # - helper for streamP4Files
2170
2171     def streamOneP4File(self, file, contents):
2172         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2173         if verbose:
2174             sys.stderr.write("%s\n" % relPath)
2175
2176         (type_base, type_mods) = split_p4_type(file["type"])
2177
2178         git_mode = "100644"
2179         if "x" in type_mods:
2180             git_mode = "100755"
2181         if type_base == "symlink":
2182             git_mode = "120000"
2183             # p4 print on a symlink sometimes contains "target\n";
2184             # if it does, remove the newline
2185             data = ''.join(contents)
2186             if data[-1] == '\n':
2187                 contents = [data[:-1]]
2188             else:
2189                 contents = [data]
2190
2191         if type_base == "utf16":
2192             # p4 delivers different text in the python output to -G
2193             # than it does when using "print -o", or normal p4 client
2194             # operations.  utf16 is converted to ascii or utf8, perhaps.
2195             # But ascii text saved as -t utf16 is completely mangled.
2196             # Invoke print -o to get the real contents.
2197             #
2198             # On windows, the newlines will always be mangled by print, so put
2199             # them back too.  This is not needed to the cygwin windows version,
2200             # just the native "NT" type.
2201             #
2202             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2203             if p4_version_string().find("/NT") >= 0:
2204                 text = text.replace("\r\n", "\n")
2205             contents = [ text ]
2206
2207         if type_base == "apple":
2208             # Apple filetype files will be streamed as a concatenation of
2209             # its appledouble header and the contents.  This is useless
2210             # on both macs and non-macs.  If using "print -q -o xx", it
2211             # will create "xx" with the data, and "%xx" with the header.
2212             # This is also not very useful.
2213             #
2214             # Ideally, someday, this script can learn how to generate
2215             # appledouble files directly and import those to git, but
2216             # non-mac machines can never find a use for apple filetype.
2217             print "\nIgnoring apple filetype file %s" % file['depotFile']
2218             return
2219
2220         # Note that we do not try to de-mangle keywords on utf16 files,
2221         # even though in theory somebody may want that.
2222         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2223         if pattern:
2224             regexp = re.compile(pattern, re.VERBOSE)
2225             text = ''.join(contents)
2226             text = regexp.sub(r'$\1$', text)
2227             contents = [ text ]
2228
2229         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2230
2231         # total length...
2232         length = 0
2233         for d in contents:
2234             length = length + len(d)
2235
2236         self.gitStream.write("data %d\n" % length)
2237         for d in contents:
2238             self.gitStream.write(d)
2239         self.gitStream.write("\n")
2240
2241     def streamOneP4Deletion(self, file):
2242         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2243         if verbose:
2244             sys.stderr.write("delete %s\n" % relPath)
2245         self.gitStream.write("D %s\n" % relPath)
2246
2247     # handle another chunk of streaming data
2248     def streamP4FilesCb(self, marshalled):
2249
2250         # catch p4 errors and complain
2251         err = None
2252         if "code" in marshalled:
2253             if marshalled["code"] == "error":
2254                 if "data" in marshalled:
2255                     err = marshalled["data"].rstrip()
2256         if err:
2257             f = None
2258             if self.stream_have_file_info:
2259                 if "depotFile" in self.stream_file:
2260                     f = self.stream_file["depotFile"]
2261             # force a failure in fast-import, else an empty
2262             # commit will be made
2263             self.gitStream.write("\n")
2264             self.gitStream.write("die-now\n")
2265             self.gitStream.close()
2266             # ignore errors, but make sure it exits first
2267             self.importProcess.wait()
2268             if f:
2269                 die("Error from p4 print for %s: %s" % (f, err))
2270             else:
2271                 die("Error from p4 print: %s" % err)
2272
2273         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2274             # start of a new file - output the old one first
2275             self.streamOneP4File(self.stream_file, self.stream_contents)
2276             self.stream_file = {}
2277             self.stream_contents = []
2278             self.stream_have_file_info = False
2279
2280         # pick up the new file information... for the
2281         # 'data' field we need to append to our array
2282         for k in marshalled.keys():
2283             if k == 'data':
2284                 self.stream_contents.append(marshalled['data'])
2285             else:
2286                 self.stream_file[k] = marshalled[k]
2287
2288         self.stream_have_file_info = True
2289
2290     # Stream directly from "p4 files" into "git fast-import"
2291     def streamP4Files(self, files):
2292         filesForCommit = []
2293         filesToRead = []
2294         filesToDelete = []
2295
2296         for f in files:
2297             # if using a client spec, only add the files that have
2298             # a path in the client
2299             if self.clientSpecDirs:
2300                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2301                     continue
2302
2303             filesForCommit.append(f)
2304             if f['action'] in self.delete_actions:
2305                 filesToDelete.append(f)
2306             else:
2307                 filesToRead.append(f)
2308
2309         # deleted files...
2310         for f in filesToDelete:
2311             self.streamOneP4Deletion(f)
2312
2313         if len(filesToRead) > 0:
2314             self.stream_file = {}
2315             self.stream_contents = []
2316             self.stream_have_file_info = False
2317
2318             # curry self argument
2319             def streamP4FilesCbSelf(entry):
2320                 self.streamP4FilesCb(entry)
2321
2322             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2323
2324             p4CmdList(["-x", "-", "print"],
2325                       stdin=fileArgs,
2326                       cb=streamP4FilesCbSelf)
2327
2328             # do the last chunk
2329             if self.stream_file.has_key('depotFile'):
2330                 self.streamOneP4File(self.stream_file, self.stream_contents)
2331
2332     def make_email(self, userid):
2333         if userid in self.users:
2334             return self.users[userid]
2335         else:
2336             return "%s <a@b>" % userid
2337
2338     # Stream a p4 tag
2339     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2340         if verbose:
2341             print "writing tag %s for commit %s" % (labelName, commit)
2342         gitStream.write("tag %s\n" % labelName)
2343         gitStream.write("from %s\n" % commit)
2344
2345         if labelDetails.has_key('Owner'):
2346             owner = labelDetails["Owner"]
2347         else:
2348             owner = None
2349
2350         # Try to use the owner of the p4 label, or failing that,
2351         # the current p4 user id.
2352         if owner:
2353             email = self.make_email(owner)
2354         else:
2355             email = self.make_email(self.p4UserId())
2356         tagger = "%s %s %s" % (email, epoch, self.tz)
2357
2358         gitStream.write("tagger %s\n" % tagger)
2359
2360         print "labelDetails=",labelDetails
2361         if labelDetails.has_key('Description'):
2362             description = labelDetails['Description']
2363         else:
2364             description = 'Label from git p4'
2365
2366         gitStream.write("data %d\n" % len(description))
2367         gitStream.write(description)
2368         gitStream.write("\n")
2369
2370     def commit(self, details, files, branch, parent = ""):
2371         epoch = details["time"]
2372         author = details["user"]
2373
2374         if self.verbose:
2375             print "commit into %s" % branch
2376
2377         # start with reading files; if that fails, we should not
2378         # create a commit.
2379         new_files = []
2380         for f in files:
2381             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2382                 new_files.append (f)
2383             else:
2384                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2385
2386         self.gitStream.write("commit %s\n" % branch)
2387 #        gitStream.write("mark :%s\n" % details["change"])
2388         self.committedChanges.add(int(details["change"]))
2389         committer = ""
2390         if author not in self.users:
2391             self.getUserMapFromPerforceServer()
2392         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2393
2394         self.gitStream.write("committer %s\n" % committer)
2395
2396         self.gitStream.write("data <<EOT\n")
2397         self.gitStream.write(details["desc"])
2398         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2399                              (','.join(self.branchPrefixes), details["change"]))
2400         if len(details['options']) > 0:
2401             self.gitStream.write(": options = %s" % details['options'])
2402         self.gitStream.write("]\nEOT\n\n")
2403
2404         if len(parent) > 0:
2405             if self.verbose:
2406                 print "parent %s" % parent
2407             self.gitStream.write("from %s\n" % parent)
2408
2409         self.streamP4Files(new_files)
2410         self.gitStream.write("\n")
2411
2412         change = int(details["change"])
2413
2414         if self.labels.has_key(change):
2415             label = self.labels[change]
2416             labelDetails = label[0]
2417             labelRevisions = label[1]
2418             if self.verbose:
2419                 print "Change %s is labelled %s" % (change, labelDetails)
2420
2421             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2422                                                 for p in self.branchPrefixes])
2423
2424             if len(files) == len(labelRevisions):
2425
2426                 cleanedFiles = {}
2427                 for info in files:
2428                     if info["action"] in self.delete_actions:
2429                         continue
2430                     cleanedFiles[info["depotFile"]] = info["rev"]
2431
2432                 if cleanedFiles == labelRevisions:
2433                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2434
2435                 else:
2436                     if not self.silent:
2437                         print ("Tag %s does not match with change %s: files do not match."
2438                                % (labelDetails["label"], change))
2439
2440             else:
2441                 if not self.silent:
2442                     print ("Tag %s does not match with change %s: file count is different."
2443                            % (labelDetails["label"], change))
2444
2445     # Build a dictionary of changelists and labels, for "detect-labels" option.
2446     def getLabels(self):
2447         self.labels = {}
2448
2449         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2450         if len(l) > 0 and not self.silent:
2451             print "Finding files belonging to labels in %s" % `self.depotPaths`
2452
2453         for output in l:
2454             label = output["label"]
2455             revisions = {}
2456             newestChange = 0
2457             if self.verbose:
2458                 print "Querying files for label %s" % label
2459             for file in p4CmdList(["files"] +
2460                                       ["%s...@%s" % (p, label)
2461                                           for p in self.depotPaths]):
2462                 revisions[file["depotFile"]] = file["rev"]
2463                 change = int(file["change"])
2464                 if change > newestChange:
2465                     newestChange = change
2466
2467             self.labels[newestChange] = [output, revisions]
2468
2469         if self.verbose:
2470             print "Label changes: %s" % self.labels.keys()
2471
2472     # Import p4 labels as git tags. A direct mapping does not
2473     # exist, so assume that if all the files are at the same revision
2474     # then we can use that, or it's something more complicated we should
2475     # just ignore.
2476     def importP4Labels(self, stream, p4Labels):
2477         if verbose:
2478             print "import p4 labels: " + ' '.join(p4Labels)
2479
2480         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2481         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2482         if len(validLabelRegexp) == 0:
2483             validLabelRegexp = defaultLabelRegexp
2484         m = re.compile(validLabelRegexp)
2485
2486         for name in p4Labels:
2487             commitFound = False
2488
2489             if not m.match(name):
2490                 if verbose:
2491                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2492                 continue
2493
2494             if name in ignoredP4Labels:
2495                 continue
2496
2497             labelDetails = p4CmdList(['label', "-o", name])[0]
2498
2499             # get the most recent changelist for each file in this label
2500             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2501                                 for p in self.depotPaths])
2502
2503             if change.has_key('change'):
2504                 # find the corresponding git commit; take the oldest commit
2505                 changelist = int(change['change'])
2506                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2507                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2508                 if len(gitCommit) == 0:
2509                     print "could not find git commit for changelist %d" % changelist
2510                 else:
2511                     gitCommit = gitCommit.strip()
2512                     commitFound = True
2513                     # Convert from p4 time format
2514                     try:
2515                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2516                     except ValueError:
2517                         print "Could not convert label time %s" % labelDetails['Update']
2518                         tmwhen = 1
2519
2520                     when = int(time.mktime(tmwhen))
2521                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2522                     if verbose:
2523                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2524             else:
2525                 if verbose:
2526                     print "Label %s has no changelists - possibly deleted?" % name
2527
2528             if not commitFound:
2529                 # We can't import this label; don't try again as it will get very
2530                 # expensive repeatedly fetching all the files for labels that will
2531                 # never be imported. If the label is moved in the future, the
2532                 # ignore will need to be removed manually.
2533                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2534
2535     def guessProjectName(self):
2536         for p in self.depotPaths:
2537             if p.endswith("/"):
2538                 p = p[:-1]
2539             p = p[p.strip().rfind("/") + 1:]
2540             if not p.endswith("/"):
2541                p += "/"
2542             return p
2543
2544     def getBranchMapping(self):
2545         lostAndFoundBranches = set()
2546
2547         user = gitConfig("git-p4.branchUser")
2548         if len(user) > 0:
2549             command = "branches -u %s" % user
2550         else:
2551             command = "branches"
2552
2553         for info in p4CmdList(command):
2554             details = p4Cmd(["branch", "-o", info["branch"]])
2555             viewIdx = 0
2556             while details.has_key("View%s" % viewIdx):
2557                 paths = details["View%s" % viewIdx].split(" ")
2558                 viewIdx = viewIdx + 1
2559                 # require standard //depot/foo/... //depot/bar/... mapping
2560                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2561                     continue
2562                 source = paths[0]
2563                 destination = paths[1]
2564                 ## HACK
2565                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2566                     source = source[len(self.depotPaths[0]):-4]
2567                     destination = destination[len(self.depotPaths[0]):-4]
2568
2569                     if destination in self.knownBranches:
2570                         if not self.silent:
2571                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2572                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2573                         continue
2574
2575                     self.knownBranches[destination] = source
2576
2577                     lostAndFoundBranches.discard(destination)
2578
2579                     if source not in self.knownBranches:
2580                         lostAndFoundBranches.add(source)
2581
2582         # Perforce does not strictly require branches to be defined, so we also
2583         # check git config for a branch list.
2584         #
2585         # Example of branch definition in git config file:
2586         # [git-p4]
2587         #   branchList=main:branchA
2588         #   branchList=main:branchB
2589         #   branchList=branchA:branchC
2590         configBranches = gitConfigList("git-p4.branchList")
2591         for branch in configBranches:
2592             if branch:
2593                 (source, destination) = branch.split(":")
2594                 self.knownBranches[destination] = source
2595
2596                 lostAndFoundBranches.discard(destination)
2597
2598                 if source not in self.knownBranches:
2599                     lostAndFoundBranches.add(source)
2600
2601
2602         for branch in lostAndFoundBranches:
2603             self.knownBranches[branch] = branch
2604
2605     def getBranchMappingFromGitBranches(self):
2606         branches = p4BranchesInGit(self.importIntoRemotes)
2607         for branch in branches.keys():
2608             if branch == "master":
2609                 branch = "main"
2610             else:
2611                 branch = branch[len(self.projectName):]
2612             self.knownBranches[branch] = branch
2613
2614     def updateOptionDict(self, d):
2615         option_keys = {}
2616         if self.keepRepoPath:
2617             option_keys['keepRepoPath'] = 1
2618
2619         d["options"] = ' '.join(sorted(option_keys.keys()))
2620
2621     def readOptions(self, d):
2622         self.keepRepoPath = (d.has_key('options')
2623                              and ('keepRepoPath' in d['options']))
2624
2625     def gitRefForBranch(self, branch):
2626         if branch == "main":
2627             return self.refPrefix + "master"
2628
2629         if len(branch) <= 0:
2630             return branch
2631
2632         return self.refPrefix + self.projectName + branch
2633
2634     def gitCommitByP4Change(self, ref, change):
2635         if self.verbose:
2636             print "looking in ref " + ref + " for change %s using bisect..." % change
2637
2638         earliestCommit = ""
2639         latestCommit = parseRevision(ref)
2640
2641         while True:
2642             if self.verbose:
2643                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2644             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2645             if len(next) == 0:
2646                 if self.verbose:
2647                     print "argh"
2648                 return ""
2649             log = extractLogMessageFromGitCommit(next)
2650             settings = extractSettingsGitLog(log)
2651             currentChange = int(settings['change'])
2652             if self.verbose:
2653                 print "current change %s" % currentChange
2654
2655             if currentChange == change:
2656                 if self.verbose:
2657                     print "found %s" % next
2658                 return next
2659
2660             if currentChange < change:
2661                 earliestCommit = "^%s" % next
2662             else:
2663                 latestCommit = "%s" % next
2664
2665         return ""
2666
2667     def importNewBranch(self, branch, maxChange):
2668         # make fast-import flush all changes to disk and update the refs using the checkpoint
2669         # command so that we can try to find the branch parent in the git history
2670         self.gitStream.write("checkpoint\n\n");
2671         self.gitStream.flush();
2672         branchPrefix = self.depotPaths[0] + branch + "/"
2673         range = "@1,%s" % maxChange
2674         #print "prefix" + branchPrefix
2675         changes = p4ChangesForPaths([branchPrefix], range)
2676         if len(changes) <= 0:
2677             return False
2678         firstChange = changes[0]
2679         #print "first change in branch: %s" % firstChange
2680         sourceBranch = self.knownBranches[branch]
2681         sourceDepotPath = self.depotPaths[0] + sourceBranch
2682         sourceRef = self.gitRefForBranch(sourceBranch)
2683         #print "source " + sourceBranch
2684
2685         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2686         #print "branch parent: %s" % branchParentChange
2687         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2688         if len(gitParent) > 0:
2689             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2690             #print "parent git commit: %s" % gitParent
2691
2692         self.importChanges(changes)
2693         return True
2694
2695     def searchParent(self, parent, branch, target):
2696         parentFound = False
2697         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2698                                      "--no-merges", parent]):
2699             blob = blob.strip()
2700             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2701                 parentFound = True
2702                 if self.verbose:
2703                     print "Found parent of %s in commit %s" % (branch, blob)
2704                 break
2705         if parentFound:
2706             return blob
2707         else:
2708             return None
2709
2710     def importChanges(self, changes):
2711         cnt = 1
2712         for change in changes:
2713             description = p4_describe(change)
2714             self.updateOptionDict(description)
2715
2716             if not self.silent:
2717                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2718                 sys.stdout.flush()
2719             cnt = cnt + 1
2720
2721             try:
2722                 if self.detectBranches:
2723                     branches = self.splitFilesIntoBranches(description)
2724                     for branch in branches.keys():
2725                         ## HACK  --hwn
2726                         branchPrefix = self.depotPaths[0] + branch + "/"
2727                         self.branchPrefixes = [ branchPrefix ]
2728
2729                         parent = ""
2730
2731                         filesForCommit = branches[branch]
2732
2733                         if self.verbose:
2734                             print "branch is %s" % branch
2735
2736                         self.updatedBranches.add(branch)
2737
2738                         if branch not in self.createdBranches:
2739                             self.createdBranches.add(branch)
2740                             parent = self.knownBranches[branch]
2741                             if parent == branch:
2742                                 parent = ""
2743                             else:
2744                                 fullBranch = self.projectName + branch
2745                                 if fullBranch not in self.p4BranchesInGit:
2746                                     if not self.silent:
2747                                         print("\n    Importing new branch %s" % fullBranch);
2748                                     if self.importNewBranch(branch, change - 1):
2749                                         parent = ""
2750                                         self.p4BranchesInGit.append(fullBranch)
2751                                     if not self.silent:
2752                                         print("\n    Resuming with change %s" % change);
2753
2754                                 if self.verbose:
2755                                     print "parent determined through known branches: %s" % parent
2756
2757                         branch = self.gitRefForBranch(branch)
2758                         parent = self.gitRefForBranch(parent)
2759
2760                         if self.verbose:
2761                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2762
2763                         if len(parent) == 0 and branch in self.initialParents:
2764                             parent = self.initialParents[branch]
2765                             del self.initialParents[branch]
2766
2767                         blob = None
2768                         if len(parent) > 0:
2769                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2770                             if self.verbose:
2771                                 print "Creating temporary branch: " + tempBranch
2772                             self.commit(description, filesForCommit, tempBranch)
2773                             self.tempBranches.append(tempBranch)
2774                             self.checkpoint()
2775                             blob = self.searchParent(parent, branch, tempBranch)
2776                         if blob:
2777                             self.commit(description, filesForCommit, branch, blob)
2778                         else:
2779                             if self.verbose:
2780                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2781                             self.commit(description, filesForCommit, branch, parent)
2782                 else:
2783                     files = self.extractFilesFromCommit(description)
2784                     self.commit(description, files, self.branch,
2785                                 self.initialParent)
2786                     # only needed once, to connect to the previous commit
2787                     self.initialParent = ""
2788             except IOError:
2789                 print self.gitError.read()
2790                 sys.exit(1)
2791
2792     def importHeadRevision(self, revision):
2793         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2794
2795         details = {}
2796         details["user"] = "git perforce import user"
2797         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2798                            % (' '.join(self.depotPaths), revision))
2799         details["change"] = revision
2800         newestRevision = 0
2801
2802         fileCnt = 0
2803         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2804
2805         for info in p4CmdList(["files"] + fileArgs):
2806
2807             if 'code' in info and info['code'] == 'error':
2808                 sys.stderr.write("p4 returned an error: %s\n"
2809                                  % info['data'])
2810                 if info['data'].find("must refer to client") >= 0:
2811                     sys.stderr.write("This particular p4 error is misleading.\n")
2812                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2813                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2814                 sys.exit(1)
2815             if 'p4ExitCode' in info:
2816                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2817                 sys.exit(1)
2818
2819
2820             change = int(info["change"])
2821             if change > newestRevision:
2822                 newestRevision = change
2823
2824             if info["action"] in self.delete_actions:
2825                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2826                 #fileCnt = fileCnt + 1
2827                 continue
2828
2829             for prop in ["depotFile", "rev", "action", "type" ]:
2830                 details["%s%s" % (prop, fileCnt)] = info[prop]
2831
2832             fileCnt = fileCnt + 1
2833
2834         details["change"] = newestRevision
2835
2836         # Use time from top-most change so that all git p4 clones of
2837         # the same p4 repo have the same commit SHA1s.
2838         res = p4_describe(newestRevision)
2839         details["time"] = res["time"]
2840
2841         self.updateOptionDict(details)
2842         try:
2843             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2844         except IOError:
2845             print "IO error with git fast-import. Is your git version recent enough?"
2846             print self.gitError.read()
2847
2848
2849     def run(self, args):
2850         self.depotPaths = []
2851         self.changeRange = ""
2852         self.previousDepotPaths = []
2853         self.hasOrigin = False
2854
2855         # map from branch depot path to parent branch
2856         self.knownBranches = {}
2857         self.initialParents = {}
2858
2859         if self.importIntoRemotes:
2860             self.refPrefix = "refs/remotes/p4/"
2861         else:
2862             self.refPrefix = "refs/heads/p4/"
2863
2864         if self.syncWithOrigin:
2865             self.hasOrigin = originP4BranchesExist()
2866             if self.hasOrigin:
2867                 if not self.silent:
2868                     print 'Syncing with origin first, using "git fetch origin"'
2869                 system("git fetch origin")
2870
2871         branch_arg_given = bool(self.branch)
2872         if len(self.branch) == 0:
2873             self.branch = self.refPrefix + "master"
2874             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2875                 system("git update-ref %s refs/heads/p4" % self.branch)
2876                 system("git branch -D p4")
2877
2878         # accept either the command-line option, or the configuration variable
2879         if self.useClientSpec:
2880             # will use this after clone to set the variable
2881             self.useClientSpec_from_options = True
2882         else:
2883             if gitConfigBool("git-p4.useclientspec"):
2884                 self.useClientSpec = True
2885         if self.useClientSpec:
2886             self.clientSpecDirs = getClientSpec()
2887
2888         # TODO: should always look at previous commits,
2889         # merge with previous imports, if possible.
2890         if args == []:
2891             if self.hasOrigin:
2892                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2893
2894             # branches holds mapping from branch name to sha1
2895             branches = p4BranchesInGit(self.importIntoRemotes)
2896
2897             # restrict to just this one, disabling detect-branches
2898             if branch_arg_given:
2899                 short = self.branch.split("/")[-1]
2900                 if short in branches:
2901                     self.p4BranchesInGit = [ short ]
2902             else:
2903                 self.p4BranchesInGit = branches.keys()
2904
2905             if len(self.p4BranchesInGit) > 1:
2906                 if not self.silent:
2907                     print "Importing from/into multiple branches"
2908                 self.detectBranches = True
2909                 for branch in branches.keys():
2910                     self.initialParents[self.refPrefix + branch] = \
2911                         branches[branch]
2912
2913             if self.verbose:
2914                 print "branches: %s" % self.p4BranchesInGit
2915
2916             p4Change = 0
2917             for branch in self.p4BranchesInGit:
2918                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2919
2920                 settings = extractSettingsGitLog(logMsg)
2921
2922                 self.readOptions(settings)
2923                 if (settings.has_key('depot-paths')
2924                     and settings.has_key ('change')):
2925                     change = int(settings['change']) + 1
2926                     p4Change = max(p4Change, change)
2927
2928                     depotPaths = sorted(settings['depot-paths'])
2929                     if self.previousDepotPaths == []:
2930                         self.previousDepotPaths = depotPaths
2931                     else:
2932                         paths = []
2933                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2934                             prev_list = prev.split("/")
2935                             cur_list = cur.split("/")
2936                             for i in range(0, min(len(cur_list), len(prev_list))):
2937                                 if cur_list[i] <> prev_list[i]:
2938                                     i = i - 1
2939                                     break
2940
2941                             paths.append ("/".join(cur_list[:i + 1]))
2942
2943                         self.previousDepotPaths = paths
2944
2945             if p4Change > 0:
2946                 self.depotPaths = sorted(self.previousDepotPaths)
2947                 self.changeRange = "@%s,#head" % p4Change
2948                 if not self.silent and not self.detectBranches:
2949                     print "Performing incremental import into %s git branch" % self.branch
2950
2951         # accept multiple ref name abbreviations:
2952         #    refs/foo/bar/branch -> use it exactly
2953         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2954         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2955         if not self.branch.startswith("refs/"):
2956             if self.importIntoRemotes:
2957                 prepend = "refs/remotes/"
2958             else:
2959                 prepend = "refs/heads/"
2960             if not self.branch.startswith("p4/"):
2961                 prepend += "p4/"
2962             self.branch = prepend + self.branch
2963
2964         if len(args) == 0 and self.depotPaths:
2965             if not self.silent:
2966                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2967         else:
2968             if self.depotPaths and self.depotPaths != args:
2969                 print ("previous import used depot path %s and now %s was specified. "
2970                        "This doesn't work!" % (' '.join (self.depotPaths),
2971                                                ' '.join (args)))
2972                 sys.exit(1)
2973
2974             self.depotPaths = sorted(args)
2975
2976         revision = ""
2977         self.users = {}
2978
2979         # Make sure no revision specifiers are used when --changesfile
2980         # is specified.
2981         bad_changesfile = False
2982         if len(self.changesFile) > 0:
2983             for p in self.depotPaths:
2984                 if p.find("@") >= 0 or p.find("#") >= 0:
2985                     bad_changesfile = True
2986                     break
2987         if bad_changesfile:
2988             die("Option --changesfile is incompatible with revision specifiers")
2989
2990         newPaths = []
2991         for p in self.depotPaths:
2992             if p.find("@") != -1:
2993                 atIdx = p.index("@")
2994                 self.changeRange = p[atIdx:]
2995                 if self.changeRange == "@all":
2996                     self.changeRange = ""
2997                 elif ',' not in self.changeRange:
2998                     revision = self.changeRange
2999                     self.changeRange = ""
3000                 p = p[:atIdx]
3001             elif p.find("#") != -1:
3002                 hashIdx = p.index("#")
3003                 revision = p[hashIdx:]
3004                 p = p[:hashIdx]
3005             elif self.previousDepotPaths == []:
3006                 # pay attention to changesfile, if given, else import
3007                 # the entire p4 tree at the head revision
3008                 if len(self.changesFile) == 0:
3009                     revision = "#head"
3010
3011             p = re.sub ("\.\.\.$", "", p)
3012             if not p.endswith("/"):
3013                 p += "/"
3014
3015             newPaths.append(p)
3016
3017         self.depotPaths = newPaths
3018
3019         # --detect-branches may change this for each branch
3020         self.branchPrefixes = self.depotPaths
3021
3022         self.loadUserMapFromCache()
3023         self.labels = {}
3024         if self.detectLabels:
3025             self.getLabels();
3026
3027         if self.detectBranches:
3028             ## FIXME - what's a P4 projectName ?
3029             self.projectName = self.guessProjectName()
3030
3031             if self.hasOrigin:
3032                 self.getBranchMappingFromGitBranches()
3033             else:
3034                 self.getBranchMapping()
3035             if self.verbose:
3036                 print "p4-git branches: %s" % self.p4BranchesInGit
3037                 print "initial parents: %s" % self.initialParents
3038             for b in self.p4BranchesInGit:
3039                 if b != "master":
3040
3041                     ## FIXME
3042                     b = b[len(self.projectName):]
3043                 self.createdBranches.add(b)
3044
3045         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3046
3047         self.importProcess = subprocess.Popen(["git", "fast-import"],
3048                                               stdin=subprocess.PIPE,
3049                                               stdout=subprocess.PIPE,
3050                                               stderr=subprocess.PIPE);
3051         self.gitOutput = self.importProcess.stdout
3052         self.gitStream = self.importProcess.stdin
3053         self.gitError = self.importProcess.stderr
3054
3055         if revision:
3056             self.importHeadRevision(revision)
3057         else:
3058             changes = []
3059
3060             if len(self.changesFile) > 0:
3061                 output = open(self.changesFile).readlines()
3062                 changeSet = set()
3063                 for line in output:
3064                     changeSet.add(int(line))
3065
3066                 for change in changeSet:
3067                     changes.append(change)
3068
3069                 changes.sort()
3070             else:
3071                 # catch "git p4 sync" with no new branches, in a repo that
3072                 # does not have any existing p4 branches
3073                 if len(args) == 0:
3074                     if not self.p4BranchesInGit:
3075                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3076
3077                     # The default branch is master, unless --branch is used to
3078                     # specify something else.  Make sure it exists, or complain
3079                     # nicely about how to use --branch.
3080                     if not self.detectBranches:
3081                         if not branch_exists(self.branch):
3082                             if branch_arg_given:
3083                                 die("Error: branch %s does not exist." % self.branch)
3084                             else:
3085                                 die("Error: no branch %s; perhaps specify one with --branch." %
3086                                     self.branch)
3087
3088                 if self.verbose:
3089                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3090                                                               self.changeRange)
3091                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
3092
3093                 if len(self.maxChanges) > 0:
3094                     changes = changes[:min(int(self.maxChanges), len(changes))]
3095
3096             if len(changes) == 0:
3097                 if not self.silent:
3098                     print "No changes to import!"
3099             else:
3100                 if not self.silent and not self.detectBranches:
3101                     print "Import destination: %s" % self.branch
3102
3103                 self.updatedBranches = set()
3104
3105                 if not self.detectBranches:
3106                     if args:
3107                         # start a new branch
3108                         self.initialParent = ""
3109                     else:
3110                         # build on a previous revision
3111                         self.initialParent = parseRevision(self.branch)
3112
3113                 self.importChanges(changes)
3114
3115                 if not self.silent:
3116                     print ""
3117                     if len(self.updatedBranches) > 0:
3118                         sys.stdout.write("Updated branches: ")
3119                         for b in self.updatedBranches:
3120                             sys.stdout.write("%s " % b)
3121                         sys.stdout.write("\n")
3122
3123         if gitConfigBool("git-p4.importLabels"):
3124             self.importLabels = True
3125
3126         if self.importLabels:
3127             p4Labels = getP4Labels(self.depotPaths)
3128             gitTags = getGitTags()
3129
3130             missingP4Labels = p4Labels - gitTags
3131             self.importP4Labels(self.gitStream, missingP4Labels)
3132
3133         self.gitStream.close()
3134         if self.importProcess.wait() != 0:
3135             die("fast-import failed: %s" % self.gitError.read())
3136         self.gitOutput.close()
3137         self.gitError.close()
3138
3139         # Cleanup temporary branches created during import
3140         if self.tempBranches != []:
3141             for branch in self.tempBranches:
3142                 read_pipe("git update-ref -d %s" % branch)
3143             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3144
3145         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3146         # a convenient shortcut refname "p4".
3147         if self.importIntoRemotes:
3148             head_ref = self.refPrefix + "HEAD"
3149             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3150                 system(["git", "symbolic-ref", head_ref, self.branch])
3151
3152         return True
3153
3154 class P4Rebase(Command):
3155     def __init__(self):
3156         Command.__init__(self)
3157         self.options = [
3158                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3159         ]
3160         self.importLabels = False
3161         self.description = ("Fetches the latest revision from perforce and "
3162                             + "rebases the current work (branch) against it")
3163
3164     def run(self, args):
3165         sync = P4Sync()
3166         sync.importLabels = self.importLabels
3167         sync.run([])
3168
3169         return self.rebase()
3170
3171     def rebase(self):
3172         if os.system("git update-index --refresh") != 0:
3173             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3174         if len(read_pipe("git diff-index HEAD --")) > 0:
3175             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3176
3177         [upstream, settings] = findUpstreamBranchPoint()
3178         if len(upstream) == 0:
3179             die("Cannot find upstream branchpoint for rebase")
3180
3181         # the branchpoint may be p4/foo~3, so strip off the parent
3182         upstream = re.sub("~[0-9]+$", "", upstream)
3183
3184         print "Rebasing the current branch onto %s" % upstream
3185         oldHead = read_pipe("git rev-parse HEAD").strip()
3186         system("git rebase %s" % upstream)
3187         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3188         return True
3189
3190 class P4Clone(P4Sync):
3191     def __init__(self):
3192         P4Sync.__init__(self)
3193         self.description = "Creates a new git repository and imports from Perforce into it"
3194         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3195         self.options += [
3196             optparse.make_option("--destination", dest="cloneDestination",
3197                                  action='store', default=None,
3198                                  help="where to leave result of the clone"),
3199             optparse.make_option("-/", dest="cloneExclude",
3200                                  action="append", type="string",
3201                                  help="exclude depot path"),
3202             optparse.make_option("--bare", dest="cloneBare",
3203                                  action="store_true", default=False),
3204         ]
3205         self.cloneDestination = None
3206         self.needsGit = False
3207         self.cloneBare = False
3208
3209     # This is required for the "append" cloneExclude action
3210     def ensure_value(self, attr, value):
3211         if not hasattr(self, attr) or getattr(self, attr) is None:
3212             setattr(self, attr, value)
3213         return getattr(self, attr)
3214
3215     def defaultDestination(self, args):
3216         ## TODO: use common prefix of args?
3217         depotPath = args[0]
3218         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3219         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3220         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3221         depotDir = re.sub(r"/$", "", depotDir)
3222         return os.path.split(depotDir)[1]
3223
3224     def run(self, args):
3225         if len(args) < 1:
3226             return False
3227
3228         if self.keepRepoPath and not self.cloneDestination:
3229             sys.stderr.write("Must specify destination for --keep-path\n")
3230             sys.exit(1)
3231
3232         depotPaths = args
3233
3234         if not self.cloneDestination and len(depotPaths) > 1:
3235             self.cloneDestination = depotPaths[-1]
3236             depotPaths = depotPaths[:-1]
3237
3238         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3239         for p in depotPaths:
3240             if not p.startswith("//"):
3241                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3242                 return False
3243
3244         if not self.cloneDestination:
3245             self.cloneDestination = self.defaultDestination(args)
3246
3247         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3248
3249         if not os.path.exists(self.cloneDestination):
3250             os.makedirs(self.cloneDestination)
3251         chdir(self.cloneDestination)
3252
3253         init_cmd = [ "git", "init" ]
3254         if self.cloneBare:
3255             init_cmd.append("--bare")
3256         retcode = subprocess.call(init_cmd)
3257         if retcode:
3258             raise CalledProcessError(retcode, init_cmd)
3259
3260         if not P4Sync.run(self, depotPaths):
3261             return False
3262
3263         # create a master branch and check out a work tree
3264         if gitBranchExists(self.branch):
3265             system([ "git", "branch", "master", self.branch ])
3266             if not self.cloneBare:
3267                 system([ "git", "checkout", "-f" ])
3268         else:
3269             print 'Not checking out any branch, use ' \
3270                   '"git checkout -q -b master <branch>"'
3271
3272         # auto-set this variable if invoked with --use-client-spec
3273         if self.useClientSpec_from_options:
3274             system("git config --bool git-p4.useclientspec true")
3275
3276         return True
3277
3278 class P4Branches(Command):
3279     def __init__(self):
3280         Command.__init__(self)
3281         self.options = [ ]
3282         self.description = ("Shows the git branches that hold imports and their "
3283                             + "corresponding perforce depot paths")
3284         self.verbose = False
3285
3286     def run(self, args):
3287         if originP4BranchesExist():
3288             createOrUpdateBranchesFromOrigin()
3289
3290         cmdline = "git rev-parse --symbolic "
3291         cmdline += " --remotes"
3292
3293         for line in read_pipe_lines(cmdline):
3294             line = line.strip()
3295
3296             if not line.startswith('p4/') or line == "p4/HEAD":
3297                 continue
3298             branch = line
3299
3300             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3301             settings = extractSettingsGitLog(log)
3302
3303             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3304         return True
3305
3306 class HelpFormatter(optparse.IndentedHelpFormatter):
3307     def __init__(self):
3308         optparse.IndentedHelpFormatter.__init__(self)
3309
3310     def format_description(self, description):
3311         if description:
3312             return description + "\n"
3313         else:
3314             return ""
3315
3316 def printUsage(commands):
3317     print "usage: %s <command> [options]" % sys.argv[0]
3318     print ""
3319     print "valid commands: %s" % ", ".join(commands)
3320     print ""
3321     print "Try %s <command> --help for command specific help." % sys.argv[0]
3322     print ""
3323
3324 commands = {
3325     "debug" : P4Debug,
3326     "submit" : P4Submit,
3327     "commit" : P4Submit,
3328     "sync" : P4Sync,
3329     "rebase" : P4Rebase,
3330     "clone" : P4Clone,
3331     "rollback" : P4RollBack,
3332     "branches" : P4Branches
3333 }
3334
3335
3336 def main():
3337     if len(sys.argv[1:]) == 0:
3338         printUsage(commands.keys())
3339         sys.exit(2)
3340
3341     cmdName = sys.argv[1]
3342     try:
3343         klass = commands[cmdName]
3344         cmd = klass()
3345     except KeyError:
3346         print "unknown command %s" % cmdName
3347         print ""
3348         printUsage(commands.keys())
3349         sys.exit(2)
3350
3351     options = cmd.options
3352     cmd.gitdir = os.environ.get("GIT_DIR", None)
3353
3354     args = sys.argv[2:]
3355
3356     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3357     if cmd.needsGit:
3358         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3359
3360     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3361                                    options,
3362                                    description = cmd.description,
3363                                    formatter = HelpFormatter())
3364
3365     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3366     global verbose
3367     verbose = cmd.verbose
3368     if cmd.needsGit:
3369         if cmd.gitdir == None:
3370             cmd.gitdir = os.path.abspath(".git")
3371             if not isValidGitDir(cmd.gitdir):
3372                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3373                 if os.path.exists(cmd.gitdir):
3374                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3375                     if len(cdup) > 0:
3376                         chdir(cdup);
3377
3378         if not isValidGitDir(cmd.gitdir):
3379             if isValidGitDir(cmd.gitdir + "/.git"):
3380                 cmd.gitdir += "/.git"
3381             else:
3382                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3383
3384         os.environ["GIT_DIR"] = cmd.gitdir
3385
3386     if not cmd.run(args):
3387         parser.print_help()
3388         sys.exit(2)
3389
3390
3391 if __name__ == '__main__':
3392     main()