Merge branch 'jk/sha1write-void'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27     from subprocess import CalledProcessError
28 except ImportError:
29     # from python2.7:subprocess.py
30     # Exception classes used by this module.
31     class CalledProcessError(Exception):
32         """This exception is raised when a process run by check_call() returns
33         a non-zero exit status.  The exit status will be stored in the
34         returncode attribute."""
35         def __init__(self, returncode, cmd):
36             self.returncode = returncode
37             self.cmd = cmd
38         def __str__(self):
39             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 def p4_build_cmd(cmd):
47     """Build a suitable p4 command line.
48
49     This consolidates building and returning a p4 command line into one
50     location. It means that hooking into the environment, or other configuration
51     can be done more easily.
52     """
53     real_cmd = ["p4"]
54
55     user = gitConfig("git-p4.user")
56     if len(user) > 0:
57         real_cmd += ["-u",user]
58
59     password = gitConfig("git-p4.password")
60     if len(password) > 0:
61         real_cmd += ["-P", password]
62
63     port = gitConfig("git-p4.port")
64     if len(port) > 0:
65         real_cmd += ["-p", port]
66
67     host = gitConfig("git-p4.host")
68     if len(host) > 0:
69         real_cmd += ["-H", host]
70
71     client = gitConfig("git-p4.client")
72     if len(client) > 0:
73         real_cmd += ["-c", client]
74
75
76     if isinstance(cmd,basestring):
77         real_cmd = ' '.join(real_cmd) + ' ' + cmd
78     else:
79         real_cmd += cmd
80     return real_cmd
81
82 def chdir(path, is_client_path=False):
83     """Do chdir to the given path, and set the PWD environment
84        variable for use by P4.  It does not look at getcwd() output.
85        Since we're not using the shell, it is necessary to set the
86        PWD environment variable explicitly.
87
88        Normally, expand the path to force it to be absolute.  This
89        addresses the use of relative path names inside P4 settings,
90        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
91        as given; it looks for .p4config using PWD.
92
93        If is_client_path, the path was handed to us directly by p4,
94        and may be a symbolic link.  Do not call os.getcwd() in this
95        case, because it will cause p4 to think that PWD is not inside
96        the client path.
97        """
98
99     os.chdir(path)
100     if not is_client_path:
101         path = os.getcwd()
102     os.environ['PWD'] = path
103
104 def die(msg):
105     if verbose:
106         raise Exception(msg)
107     else:
108         sys.stderr.write(msg + "\n")
109         sys.exit(1)
110
111 def write_pipe(c, stdin):
112     if verbose:
113         sys.stderr.write('Writing pipe: %s\n' % str(c))
114
115     expand = isinstance(c,basestring)
116     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117     pipe = p.stdin
118     val = pipe.write(stdin)
119     pipe.close()
120     if p.wait():
121         die('Command failed: %s' % str(c))
122
123     return val
124
125 def p4_write_pipe(c, stdin):
126     real_cmd = p4_build_cmd(c)
127     return write_pipe(real_cmd, stdin)
128
129 def read_pipe(c, ignore_error=False):
130     if verbose:
131         sys.stderr.write('Reading pipe: %s\n' % str(c))
132
133     expand = isinstance(c,basestring)
134     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135     pipe = p.stdout
136     val = pipe.read()
137     if p.wait() and not ignore_error:
138         die('Command failed: %s' % str(c))
139
140     return val
141
142 def p4_read_pipe(c, ignore_error=False):
143     real_cmd = p4_build_cmd(c)
144     return read_pipe(real_cmd, ignore_error)
145
146 def read_pipe_lines(c):
147     if verbose:
148         sys.stderr.write('Reading pipe: %s\n' % str(c))
149
150     expand = isinstance(c, basestring)
151     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152     pipe = p.stdout
153     val = pipe.readlines()
154     if pipe.close() or p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_read_pipe_lines(c):
160     """Specifically invoke p4 on the command supplied. """
161     real_cmd = p4_build_cmd(c)
162     return read_pipe_lines(real_cmd)
163
164 def p4_has_command(cmd):
165     """Ask p4 for help on this command.  If it returns an error, the
166        command does not exist in this version of p4."""
167     real_cmd = p4_build_cmd(["help", cmd])
168     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169                                    stderr=subprocess.PIPE)
170     p.communicate()
171     return p.returncode == 0
172
173 def p4_has_move_command():
174     """See if the move command exists, that it supports -k, and that
175        it has not been administratively disabled.  The arguments
176        must be correct, but the filenames do not have to exist.  Use
177        ones with wildcards so even if they exist, it will fail."""
178
179     if not p4_has_command("move"):
180         return False
181     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183     (out, err) = p.communicate()
184     # return code will be 1 in either case
185     if err.find("Invalid option") >= 0:
186         return False
187     if err.find("disabled") >= 0:
188         return False
189     # assume it failed because @... was invalid changelist
190     return True
191
192 def system(cmd):
193     expand = isinstance(cmd,basestring)
194     if verbose:
195         sys.stderr.write("executing %s\n" % str(cmd))
196     retcode = subprocess.call(cmd, shell=expand)
197     if retcode:
198         raise CalledProcessError(retcode, cmd)
199
200 def p4_system(cmd):
201     """Specifically invoke p4 as the system command. """
202     real_cmd = p4_build_cmd(cmd)
203     expand = isinstance(real_cmd, basestring)
204     retcode = subprocess.call(real_cmd, shell=expand)
205     if retcode:
206         raise CalledProcessError(retcode, real_cmd)
207
208 _p4_version_string = None
209 def p4_version_string():
210     """Read the version string, showing just the last line, which
211        hopefully is the interesting version bit.
212
213        $ p4 -V
214        Perforce - The Fast Software Configuration Management System.
215        Copyright 1995-2011 Perforce Software.  All rights reserved.
216        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
217     """
218     global _p4_version_string
219     if not _p4_version_string:
220         a = p4_read_pipe_lines(["-V"])
221         _p4_version_string = a[-1].rstrip()
222     return _p4_version_string
223
224 def p4_integrate(src, dest):
225     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
226
227 def p4_sync(f, *options):
228     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
229
230 def p4_add(f):
231     # forcibly add file names with wildcards
232     if wildcard_present(f):
233         p4_system(["add", "-f", f])
234     else:
235         p4_system(["add", f])
236
237 def p4_delete(f):
238     p4_system(["delete", wildcard_encode(f)])
239
240 def p4_edit(f):
241     p4_system(["edit", wildcard_encode(f)])
242
243 def p4_revert(f):
244     p4_system(["revert", wildcard_encode(f)])
245
246 def p4_reopen(type, f):
247     p4_system(["reopen", "-t", type, wildcard_encode(f)])
248
249 def p4_move(src, dest):
250     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
251
252 def p4_describe(change):
253     """Make sure it returns a valid result by checking for
254        the presence of field "time".  Return a dict of the
255        results."""
256
257     ds = p4CmdList(["describe", "-s", str(change)])
258     if len(ds) != 1:
259         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
260
261     d = ds[0]
262
263     if "p4ExitCode" in d:
264         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265                                                       str(d)))
266     if "code" in d:
267         if d["code"] == "error":
268             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
269
270     if "time" not in d:
271         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
272
273     return d
274
275 #
276 # Canonicalize the p4 type and return a tuple of the
277 # base type, plus any modifiers.  See "p4 help filetypes"
278 # for a list and explanation.
279 #
280 def split_p4_type(p4type):
281
282     p4_filetypes_historical = {
283         "ctempobj": "binary+Sw",
284         "ctext": "text+C",
285         "cxtext": "text+Cx",
286         "ktext": "text+k",
287         "kxtext": "text+kx",
288         "ltext": "text+F",
289         "tempobj": "binary+FSw",
290         "ubinary": "binary+F",
291         "uresource": "resource+F",
292         "uxbinary": "binary+Fx",
293         "xbinary": "binary+x",
294         "xltext": "text+Fx",
295         "xtempobj": "binary+Swx",
296         "xtext": "text+x",
297         "xunicode": "unicode+x",
298         "xutf16": "utf16+x",
299     }
300     if p4type in p4_filetypes_historical:
301         p4type = p4_filetypes_historical[p4type]
302     mods = ""
303     s = p4type.split("+")
304     base = s[0]
305     mods = ""
306     if len(s) > 1:
307         mods = s[1]
308     return (base, mods)
309
310 #
311 # return the raw p4 type of a file (text, text+ko, etc)
312 #
313 def p4_type(file):
314     results = p4CmdList(["fstat", "-T", "headType", file])
315     return results[0]['headType']
316
317 #
318 # Given a type base and modifier, return a regexp matching
319 # the keywords that can be expanded in the file
320 #
321 def p4_keywords_regexp_for_type(base, type_mods):
322     if base in ("text", "unicode", "binary"):
323         kwords = None
324         if "ko" in type_mods:
325             kwords = 'Id|Header'
326         elif "k" in type_mods:
327             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328         else:
329             return None
330         pattern = r"""
331             \$              # Starts with a dollar, followed by...
332             (%s)            # one of the keywords, followed by...
333             (:[^$\n]+)?     # possibly an old expansion, followed by...
334             \$              # another dollar
335             """ % kwords
336         return pattern
337     else:
338         return None
339
340 #
341 # Given a file, return a regexp matching the possible
342 # RCS keywords that will be expanded, or None for files
343 # with kw expansion turned off.
344 #
345 def p4_keywords_regexp_for_file(file):
346     if not os.path.exists(file):
347         return None
348     else:
349         (type_base, type_mods) = split_p4_type(p4_type(file))
350         return p4_keywords_regexp_for_type(type_base, type_mods)
351
352 def setP4ExecBit(file, mode):
353     # Reopens an already open file and changes the execute bit to match
354     # the execute bit setting in the passed in mode.
355
356     p4Type = "+x"
357
358     if not isModeExec(mode):
359         p4Type = getP4OpenedType(file)
360         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362         if p4Type[-1] == "+":
363             p4Type = p4Type[0:-1]
364
365     p4_reopen(p4Type, file)
366
367 def getP4OpenedType(file):
368     # Returns the perforce file type for the given file.
369
370     result = p4_read_pipe(["opened", wildcard_encode(file)])
371     match = re.match(".*\((.+)\)\r?$", result)
372     if match:
373         return match.group(1)
374     else:
375         die("Could not determine file type for %s (result: '%s')" % (file, result))
376
377 # Return the set of all p4 labels
378 def getP4Labels(depotPaths):
379     labels = set()
380     if isinstance(depotPaths,basestring):
381         depotPaths = [depotPaths]
382
383     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384         label = l['label']
385         labels.add(label)
386
387     return labels
388
389 # Return the set of all git tags
390 def getGitTags():
391     gitTags = set()
392     for line in read_pipe_lines(["git", "tag"]):
393         tag = line.strip()
394         gitTags.add(tag)
395     return gitTags
396
397 def diffTreePattern():
398     # This is a simple generator for the diff tree regex pattern. This could be
399     # a class variable if this and parseDiffTreeEntry were a part of a class.
400     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401     while True:
402         yield pattern
403
404 def parseDiffTreeEntry(entry):
405     """Parses a single diff tree entry into its component elements.
406
407     See git-diff-tree(1) manpage for details about the format of the diff
408     output. This method returns a dictionary with the following elements:
409
410     src_mode - The mode of the source file
411     dst_mode - The mode of the destination file
412     src_sha1 - The sha1 for the source file
413     dst_sha1 - The sha1 fr the destination file
414     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415     status_score - The score for the status (applicable for 'C' and 'R'
416                    statuses). This is None if there is no score.
417     src - The path for the source file.
418     dst - The path for the destination file. This is only present for
419           copy or renames. If it is not present, this is None.
420
421     If the pattern is not matched, None is returned."""
422
423     match = diffTreePattern().next().match(entry)
424     if match:
425         return {
426             'src_mode': match.group(1),
427             'dst_mode': match.group(2),
428             'src_sha1': match.group(3),
429             'dst_sha1': match.group(4),
430             'status': match.group(5),
431             'status_score': match.group(6),
432             'src': match.group(7),
433             'dst': match.group(10)
434         }
435     return None
436
437 def isModeExec(mode):
438     # Returns True if the given git mode represents an executable file,
439     # otherwise False.
440     return mode[-3:] == "755"
441
442 def isModeExecChanged(src_mode, dst_mode):
443     return isModeExec(src_mode) != isModeExec(dst_mode)
444
445 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
446
447     if isinstance(cmd,basestring):
448         cmd = "-G " + cmd
449         expand = True
450     else:
451         cmd = ["-G"] + cmd
452         expand = False
453
454     cmd = p4_build_cmd(cmd)
455     if verbose:
456         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
457
458     # Use a temporary file to avoid deadlocks without
459     # subprocess.communicate(), which would put another copy
460     # of stdout into memory.
461     stdin_file = None
462     if stdin is not None:
463         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
464         if isinstance(stdin,basestring):
465             stdin_file.write(stdin)
466         else:
467             for i in stdin:
468                 stdin_file.write(i + '\n')
469         stdin_file.flush()
470         stdin_file.seek(0)
471
472     p4 = subprocess.Popen(cmd,
473                           shell=expand,
474                           stdin=stdin_file,
475                           stdout=subprocess.PIPE)
476
477     result = []
478     try:
479         while True:
480             entry = marshal.load(p4.stdout)
481             if cb is not None:
482                 cb(entry)
483             else:
484                 result.append(entry)
485     except EOFError:
486         pass
487     exitCode = p4.wait()
488     if exitCode != 0:
489         entry = {}
490         entry["p4ExitCode"] = exitCode
491         result.append(entry)
492
493     return result
494
495 def p4Cmd(cmd):
496     list = p4CmdList(cmd)
497     result = {}
498     for entry in list:
499         result.update(entry)
500     return result;
501
502 def p4Where(depotPath):
503     if not depotPath.endswith("/"):
504         depotPath += "/"
505     depotPath = depotPath + "..."
506     outputList = p4CmdList(["where", depotPath])
507     output = None
508     for entry in outputList:
509         if "depotFile" in entry:
510             if entry["depotFile"] == depotPath:
511                 output = entry
512                 break
513         elif "data" in entry:
514             data = entry.get("data")
515             space = data.find(" ")
516             if data[:space] == depotPath:
517                 output = entry
518                 break
519     if output == None:
520         return ""
521     if output["code"] == "error":
522         return ""
523     clientPath = ""
524     if "path" in output:
525         clientPath = output.get("path")
526     elif "data" in output:
527         data = output.get("data")
528         lastSpace = data.rfind(" ")
529         clientPath = data[lastSpace + 1:]
530
531     if clientPath.endswith("..."):
532         clientPath = clientPath[:-3]
533     return clientPath
534
535 def currentGitBranch():
536     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
537
538 def isValidGitDir(path):
539     if (os.path.exists(path + "/HEAD")
540         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
541         return True;
542     return False
543
544 def parseRevision(ref):
545     return read_pipe("git rev-parse %s" % ref).strip()
546
547 def branchExists(ref):
548     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
549                      ignore_error=True)
550     return len(rev) > 0
551
552 def extractLogMessageFromGitCommit(commit):
553     logMessage = ""
554
555     ## fixme: title is first line of commit, not 1st paragraph.
556     foundTitle = False
557     for log in read_pipe_lines("git cat-file commit %s" % commit):
558        if not foundTitle:
559            if len(log) == 1:
560                foundTitle = True
561            continue
562
563        logMessage += log
564     return logMessage
565
566 def extractSettingsGitLog(log):
567     values = {}
568     for line in log.split("\n"):
569         line = line.strip()
570         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
571         if not m:
572             continue
573
574         assignments = m.group(1).split (':')
575         for a in assignments:
576             vals = a.split ('=')
577             key = vals[0].strip()
578             val = ('='.join (vals[1:])).strip()
579             if val.endswith ('\"') and val.startswith('"'):
580                 val = val[1:-1]
581
582             values[key] = val
583
584     paths = values.get("depot-paths")
585     if not paths:
586         paths = values.get("depot-path")
587     if paths:
588         values['depot-paths'] = paths.split(',')
589     return values
590
591 def gitBranchExists(branch):
592     proc = subprocess.Popen(["git", "rev-parse", branch],
593                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
594     return proc.wait() == 0;
595
596 _gitConfig = {}
597
598 def gitConfig(key):
599     if not _gitConfig.has_key(key):
600         cmd = [ "git", "config", key ]
601         s = read_pipe(cmd, ignore_error=True)
602         _gitConfig[key] = s.strip()
603     return _gitConfig[key]
604
605 def gitConfigBool(key):
606     """Return a bool, using git config --bool.  It is True only if the
607        variable is set to true, and False if set to false or not present
608        in the config."""
609
610     if not _gitConfig.has_key(key):
611         cmd = [ "git", "config", "--bool", key ]
612         s = read_pipe(cmd, ignore_error=True)
613         v = s.strip()
614         _gitConfig[key] = v == "true"
615     return _gitConfig[key]
616
617 def gitConfigList(key):
618     if not _gitConfig.has_key(key):
619         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
620         _gitConfig[key] = s.strip().split(os.linesep)
621     return _gitConfig[key]
622
623 def p4BranchesInGit(branchesAreInRemotes=True):
624     """Find all the branches whose names start with "p4/", looking
625        in remotes or heads as specified by the argument.  Return
626        a dictionary of { branch: revision } for each one found.
627        The branch names are the short names, without any
628        "p4/" prefix."""
629
630     branches = {}
631
632     cmdline = "git rev-parse --symbolic "
633     if branchesAreInRemotes:
634         cmdline += "--remotes"
635     else:
636         cmdline += "--branches"
637
638     for line in read_pipe_lines(cmdline):
639         line = line.strip()
640
641         # only import to p4/
642         if not line.startswith('p4/'):
643             continue
644         # special symbolic ref to p4/master
645         if line == "p4/HEAD":
646             continue
647
648         # strip off p4/ prefix
649         branch = line[len("p4/"):]
650
651         branches[branch] = parseRevision(line)
652
653     return branches
654
655 def branch_exists(branch):
656     """Make sure that the given ref name really exists."""
657
658     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
659     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
660     out, _ = p.communicate()
661     if p.returncode:
662         return False
663     # expect exactly one line of output: the branch name
664     return out.rstrip() == branch
665
666 def findUpstreamBranchPoint(head = "HEAD"):
667     branches = p4BranchesInGit()
668     # map from depot-path to branch name
669     branchByDepotPath = {}
670     for branch in branches.keys():
671         tip = branches[branch]
672         log = extractLogMessageFromGitCommit(tip)
673         settings = extractSettingsGitLog(log)
674         if settings.has_key("depot-paths"):
675             paths = ",".join(settings["depot-paths"])
676             branchByDepotPath[paths] = "remotes/p4/" + branch
677
678     settings = None
679     parent = 0
680     while parent < 65535:
681         commit = head + "~%s" % parent
682         log = extractLogMessageFromGitCommit(commit)
683         settings = extractSettingsGitLog(log)
684         if settings.has_key("depot-paths"):
685             paths = ",".join(settings["depot-paths"])
686             if branchByDepotPath.has_key(paths):
687                 return [branchByDepotPath[paths], settings]
688
689         parent = parent + 1
690
691     return ["", settings]
692
693 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
694     if not silent:
695         print ("Creating/updating branch(es) in %s based on origin branch(es)"
696                % localRefPrefix)
697
698     originPrefix = "origin/p4/"
699
700     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
701         line = line.strip()
702         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
703             continue
704
705         headName = line[len(originPrefix):]
706         remoteHead = localRefPrefix + headName
707         originHead = line
708
709         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
710         if (not original.has_key('depot-paths')
711             or not original.has_key('change')):
712             continue
713
714         update = False
715         if not gitBranchExists(remoteHead):
716             if verbose:
717                 print "creating %s" % remoteHead
718             update = True
719         else:
720             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
721             if settings.has_key('change') > 0:
722                 if settings['depot-paths'] == original['depot-paths']:
723                     originP4Change = int(original['change'])
724                     p4Change = int(settings['change'])
725                     if originP4Change > p4Change:
726                         print ("%s (%s) is newer than %s (%s). "
727                                "Updating p4 branch from origin."
728                                % (originHead, originP4Change,
729                                   remoteHead, p4Change))
730                         update = True
731                 else:
732                     print ("Ignoring: %s was imported from %s while "
733                            "%s was imported from %s"
734                            % (originHead, ','.join(original['depot-paths']),
735                               remoteHead, ','.join(settings['depot-paths'])))
736
737         if update:
738             system("git update-ref %s %s" % (remoteHead, originHead))
739
740 def originP4BranchesExist():
741         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
742
743 def p4ChangesForPaths(depotPaths, changeRange):
744     assert depotPaths
745     cmd = ['changes']
746     for p in depotPaths:
747         cmd += ["%s...%s" % (p, changeRange)]
748     output = p4_read_pipe_lines(cmd)
749
750     changes = {}
751     for line in output:
752         changeNum = int(line.split(" ")[1])
753         changes[changeNum] = True
754
755     changelist = changes.keys()
756     changelist.sort()
757     return changelist
758
759 def p4PathStartsWith(path, prefix):
760     # This method tries to remedy a potential mixed-case issue:
761     #
762     # If UserA adds  //depot/DirA/file1
763     # and UserB adds //depot/dira/file2
764     #
765     # we may or may not have a problem. If you have core.ignorecase=true,
766     # we treat DirA and dira as the same directory
767     if gitConfigBool("core.ignorecase"):
768         return path.lower().startswith(prefix.lower())
769     return path.startswith(prefix)
770
771 def getClientSpec():
772     """Look at the p4 client spec, create a View() object that contains
773        all the mappings, and return it."""
774
775     specList = p4CmdList("client -o")
776     if len(specList) != 1:
777         die('Output from "client -o" is %d lines, expecting 1' %
778             len(specList))
779
780     # dictionary of all client parameters
781     entry = specList[0]
782
783     # the //client/ name
784     client_name = entry["Client"]
785
786     # just the keys that start with "View"
787     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
788
789     # hold this new View
790     view = View(client_name)
791
792     # append the lines, in order, to the view
793     for view_num in range(len(view_keys)):
794         k = "View%d" % view_num
795         if k not in view_keys:
796             die("Expected view key %s missing" % k)
797         view.append(entry[k])
798
799     return view
800
801 def getClientRoot():
802     """Grab the client directory."""
803
804     output = p4CmdList("client -o")
805     if len(output) != 1:
806         die('Output from "client -o" is %d lines, expecting 1' % len(output))
807
808     entry = output[0]
809     if "Root" not in entry:
810         die('Client has no "Root"')
811
812     return entry["Root"]
813
814 #
815 # P4 wildcards are not allowed in filenames.  P4 complains
816 # if you simply add them, but you can force it with "-f", in
817 # which case it translates them into %xx encoding internally.
818 #
819 def wildcard_decode(path):
820     # Search for and fix just these four characters.  Do % last so
821     # that fixing it does not inadvertently create new %-escapes.
822     # Cannot have * in a filename in windows; untested as to
823     # what p4 would do in such a case.
824     if not platform.system() == "Windows":
825         path = path.replace("%2A", "*")
826     path = path.replace("%23", "#") \
827                .replace("%40", "@") \
828                .replace("%25", "%")
829     return path
830
831 def wildcard_encode(path):
832     # do % first to avoid double-encoding the %s introduced here
833     path = path.replace("%", "%25") \
834                .replace("*", "%2A") \
835                .replace("#", "%23") \
836                .replace("@", "%40")
837     return path
838
839 def wildcard_present(path):
840     m = re.search("[*#@%]", path)
841     return m is not None
842
843 class Command:
844     def __init__(self):
845         self.usage = "usage: %prog [options]"
846         self.needsGit = True
847         self.verbose = False
848
849 class P4UserMap:
850     def __init__(self):
851         self.userMapFromPerforceServer = False
852         self.myP4UserId = None
853
854     def p4UserId(self):
855         if self.myP4UserId:
856             return self.myP4UserId
857
858         results = p4CmdList("user -o")
859         for r in results:
860             if r.has_key('User'):
861                 self.myP4UserId = r['User']
862                 return r['User']
863         die("Could not find your p4 user id")
864
865     def p4UserIsMe(self, p4User):
866         # return True if the given p4 user is actually me
867         me = self.p4UserId()
868         if not p4User or p4User != me:
869             return False
870         else:
871             return True
872
873     def getUserCacheFilename(self):
874         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
875         return home + "/.gitp4-usercache.txt"
876
877     def getUserMapFromPerforceServer(self):
878         if self.userMapFromPerforceServer:
879             return
880         self.users = {}
881         self.emails = {}
882
883         for output in p4CmdList("users"):
884             if not output.has_key("User"):
885                 continue
886             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
887             self.emails[output["Email"]] = output["User"]
888
889
890         s = ''
891         for (key, val) in self.users.items():
892             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
893
894         open(self.getUserCacheFilename(), "wb").write(s)
895         self.userMapFromPerforceServer = True
896
897     def loadUserMapFromCache(self):
898         self.users = {}
899         self.userMapFromPerforceServer = False
900         try:
901             cache = open(self.getUserCacheFilename(), "rb")
902             lines = cache.readlines()
903             cache.close()
904             for line in lines:
905                 entry = line.strip().split("\t")
906                 self.users[entry[0]] = entry[1]
907         except IOError:
908             self.getUserMapFromPerforceServer()
909
910 class P4Debug(Command):
911     def __init__(self):
912         Command.__init__(self)
913         self.options = []
914         self.description = "A tool to debug the output of p4 -G."
915         self.needsGit = False
916
917     def run(self, args):
918         j = 0
919         for output in p4CmdList(args):
920             print 'Element: %d' % j
921             j += 1
922             print output
923         return True
924
925 class P4RollBack(Command):
926     def __init__(self):
927         Command.__init__(self)
928         self.options = [
929             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
930         ]
931         self.description = "A tool to debug the multi-branch import. Don't use :)"
932         self.rollbackLocalBranches = False
933
934     def run(self, args):
935         if len(args) != 1:
936             return False
937         maxChange = int(args[0])
938
939         if "p4ExitCode" in p4Cmd("changes -m 1"):
940             die("Problems executing p4");
941
942         if self.rollbackLocalBranches:
943             refPrefix = "refs/heads/"
944             lines = read_pipe_lines("git rev-parse --symbolic --branches")
945         else:
946             refPrefix = "refs/remotes/"
947             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
948
949         for line in lines:
950             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
951                 line = line.strip()
952                 ref = refPrefix + line
953                 log = extractLogMessageFromGitCommit(ref)
954                 settings = extractSettingsGitLog(log)
955
956                 depotPaths = settings['depot-paths']
957                 change = settings['change']
958
959                 changed = False
960
961                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
962                                                            for p in depotPaths]))) == 0:
963                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
964                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
965                     continue
966
967                 while change and int(change) > maxChange:
968                     changed = True
969                     if self.verbose:
970                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
971                     system("git update-ref %s \"%s^\"" % (ref, ref))
972                     log = extractLogMessageFromGitCommit(ref)
973                     settings =  extractSettingsGitLog(log)
974
975
976                     depotPaths = settings['depot-paths']
977                     change = settings['change']
978
979                 if changed:
980                     print "%s rewound to %s" % (ref, change)
981
982         return True
983
984 class P4Submit(Command, P4UserMap):
985
986     conflict_behavior_choices = ("ask", "skip", "quit")
987
988     def __init__(self):
989         Command.__init__(self)
990         P4UserMap.__init__(self)
991         self.options = [
992                 optparse.make_option("--origin", dest="origin"),
993                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
994                 # preserve the user, requires relevant p4 permissions
995                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
996                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
997                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
998                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
999                 optparse.make_option("--conflict", dest="conflict_behavior",
1000                                      choices=self.conflict_behavior_choices),
1001                 optparse.make_option("--branch", dest="branch"),
1002         ]
1003         self.description = "Submit changes from git to the perforce depot."
1004         self.usage += " [name of git branch to submit into perforce depot]"
1005         self.origin = ""
1006         self.detectRenames = False
1007         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1008         self.dry_run = False
1009         self.prepare_p4_only = False
1010         self.conflict_behavior = None
1011         self.isWindows = (platform.system() == "Windows")
1012         self.exportLabels = False
1013         self.p4HasMoveCommand = p4_has_move_command()
1014         self.branch = None
1015
1016     def check(self):
1017         if len(p4CmdList("opened ...")) > 0:
1018             die("You have files opened with perforce! Close them before starting the sync.")
1019
1020     def separate_jobs_from_description(self, message):
1021         """Extract and return a possible Jobs field in the commit
1022            message.  It goes into a separate section in the p4 change
1023            specification.
1024
1025            A jobs line starts with "Jobs:" and looks like a new field
1026            in a form.  Values are white-space separated on the same
1027            line or on following lines that start with a tab.
1028
1029            This does not parse and extract the full git commit message
1030            like a p4 form.  It just sees the Jobs: line as a marker
1031            to pass everything from then on directly into the p4 form,
1032            but outside the description section.
1033
1034            Return a tuple (stripped log message, jobs string)."""
1035
1036         m = re.search(r'^Jobs:', message, re.MULTILINE)
1037         if m is None:
1038             return (message, None)
1039
1040         jobtext = message[m.start():]
1041         stripped_message = message[:m.start()].rstrip()
1042         return (stripped_message, jobtext)
1043
1044     def prepareLogMessage(self, template, message, jobs):
1045         """Edits the template returned from "p4 change -o" to insert
1046            the message in the Description field, and the jobs text in
1047            the Jobs field."""
1048         result = ""
1049
1050         inDescriptionSection = False
1051
1052         for line in template.split("\n"):
1053             if line.startswith("#"):
1054                 result += line + "\n"
1055                 continue
1056
1057             if inDescriptionSection:
1058                 if line.startswith("Files:") or line.startswith("Jobs:"):
1059                     inDescriptionSection = False
1060                     # insert Jobs section
1061                     if jobs:
1062                         result += jobs + "\n"
1063                 else:
1064                     continue
1065             else:
1066                 if line.startswith("Description:"):
1067                     inDescriptionSection = True
1068                     line += "\n"
1069                     for messageLine in message.split("\n"):
1070                         line += "\t" + messageLine + "\n"
1071
1072             result += line + "\n"
1073
1074         return result
1075
1076     def patchRCSKeywords(self, file, pattern):
1077         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1078         (handle, outFileName) = tempfile.mkstemp(dir='.')
1079         try:
1080             outFile = os.fdopen(handle, "w+")
1081             inFile = open(file, "r")
1082             regexp = re.compile(pattern, re.VERBOSE)
1083             for line in inFile.readlines():
1084                 line = regexp.sub(r'$\1$', line)
1085                 outFile.write(line)
1086             inFile.close()
1087             outFile.close()
1088             # Forcibly overwrite the original file
1089             os.unlink(file)
1090             shutil.move(outFileName, file)
1091         except:
1092             # cleanup our temporary file
1093             os.unlink(outFileName)
1094             print "Failed to strip RCS keywords in %s" % file
1095             raise
1096
1097         print "Patched up RCS keywords in %s" % file
1098
1099     def p4UserForCommit(self,id):
1100         # Return the tuple (perforce user,git email) for a given git commit id
1101         self.getUserMapFromPerforceServer()
1102         gitEmail = read_pipe(["git", "log", "--max-count=1",
1103                               "--format=%ae", id])
1104         gitEmail = gitEmail.strip()
1105         if not self.emails.has_key(gitEmail):
1106             return (None,gitEmail)
1107         else:
1108             return (self.emails[gitEmail],gitEmail)
1109
1110     def checkValidP4Users(self,commits):
1111         # check if any git authors cannot be mapped to p4 users
1112         for id in commits:
1113             (user,email) = self.p4UserForCommit(id)
1114             if not user:
1115                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1116                 if gitConfigBool("git-p4.allowMissingP4Users"):
1117                     print "%s" % msg
1118                 else:
1119                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1120
1121     def lastP4Changelist(self):
1122         # Get back the last changelist number submitted in this client spec. This
1123         # then gets used to patch up the username in the change. If the same
1124         # client spec is being used by multiple processes then this might go
1125         # wrong.
1126         results = p4CmdList("client -o")        # find the current client
1127         client = None
1128         for r in results:
1129             if r.has_key('Client'):
1130                 client = r['Client']
1131                 break
1132         if not client:
1133             die("could not get client spec")
1134         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1135         for r in results:
1136             if r.has_key('change'):
1137                 return r['change']
1138         die("Could not get changelist number for last submit - cannot patch up user details")
1139
1140     def modifyChangelistUser(self, changelist, newUser):
1141         # fixup the user field of a changelist after it has been submitted.
1142         changes = p4CmdList("change -o %s" % changelist)
1143         if len(changes) != 1:
1144             die("Bad output from p4 change modifying %s to user %s" %
1145                 (changelist, newUser))
1146
1147         c = changes[0]
1148         if c['User'] == newUser: return   # nothing to do
1149         c['User'] = newUser
1150         input = marshal.dumps(c)
1151
1152         result = p4CmdList("change -f -i", stdin=input)
1153         for r in result:
1154             if r.has_key('code'):
1155                 if r['code'] == 'error':
1156                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1157             if r.has_key('data'):
1158                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1159                 return
1160         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1161
1162     def canChangeChangelists(self):
1163         # check to see if we have p4 admin or super-user permissions, either of
1164         # which are required to modify changelists.
1165         results = p4CmdList(["protects", self.depotPath])
1166         for r in results:
1167             if r.has_key('perm'):
1168                 if r['perm'] == 'admin':
1169                     return 1
1170                 if r['perm'] == 'super':
1171                     return 1
1172         return 0
1173
1174     def prepareSubmitTemplate(self):
1175         """Run "p4 change -o" to grab a change specification template.
1176            This does not use "p4 -G", as it is nice to keep the submission
1177            template in original order, since a human might edit it.
1178
1179            Remove lines in the Files section that show changes to files
1180            outside the depot path we're committing into."""
1181
1182         template = ""
1183         inFilesSection = False
1184         for line in p4_read_pipe_lines(['change', '-o']):
1185             if line.endswith("\r\n"):
1186                 line = line[:-2] + "\n"
1187             if inFilesSection:
1188                 if line.startswith("\t"):
1189                     # path starts and ends with a tab
1190                     path = line[1:]
1191                     lastTab = path.rfind("\t")
1192                     if lastTab != -1:
1193                         path = path[:lastTab]
1194                         if not p4PathStartsWith(path, self.depotPath):
1195                             continue
1196                 else:
1197                     inFilesSection = False
1198             else:
1199                 if line.startswith("Files:"):
1200                     inFilesSection = True
1201
1202             template += line
1203
1204         return template
1205
1206     def edit_template(self, template_file):
1207         """Invoke the editor to let the user change the submission
1208            message.  Return true if okay to continue with the submit."""
1209
1210         # if configured to skip the editing part, just submit
1211         if gitConfigBool("git-p4.skipSubmitEdit"):
1212             return True
1213
1214         # look at the modification time, to check later if the user saved
1215         # the file
1216         mtime = os.stat(template_file).st_mtime
1217
1218         # invoke the editor
1219         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1220             editor = os.environ.get("P4EDITOR")
1221         else:
1222             editor = read_pipe("git var GIT_EDITOR").strip()
1223         system(editor + " " + template_file)
1224
1225         # If the file was not saved, prompt to see if this patch should
1226         # be skipped.  But skip this verification step if configured so.
1227         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1228             return True
1229
1230         # modification time updated means user saved the file
1231         if os.stat(template_file).st_mtime > mtime:
1232             return True
1233
1234         while True:
1235             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1236             if response == 'y':
1237                 return True
1238             if response == 'n':
1239                 return False
1240
1241     def applyCommit(self, id):
1242         """Apply one commit, return True if it succeeded."""
1243
1244         print "Applying", read_pipe(["git", "show", "-s",
1245                                      "--format=format:%h %s", id])
1246
1247         (p4User, gitEmail) = self.p4UserForCommit(id)
1248
1249         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1250         filesToAdd = set()
1251         filesToDelete = set()
1252         editedFiles = set()
1253         pureRenameCopy = set()
1254         filesToChangeExecBit = {}
1255
1256         for line in diff:
1257             diff = parseDiffTreeEntry(line)
1258             modifier = diff['status']
1259             path = diff['src']
1260             if modifier == "M":
1261                 p4_edit(path)
1262                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1263                     filesToChangeExecBit[path] = diff['dst_mode']
1264                 editedFiles.add(path)
1265             elif modifier == "A":
1266                 filesToAdd.add(path)
1267                 filesToChangeExecBit[path] = diff['dst_mode']
1268                 if path in filesToDelete:
1269                     filesToDelete.remove(path)
1270             elif modifier == "D":
1271                 filesToDelete.add(path)
1272                 if path in filesToAdd:
1273                     filesToAdd.remove(path)
1274             elif modifier == "C":
1275                 src, dest = diff['src'], diff['dst']
1276                 p4_integrate(src, dest)
1277                 pureRenameCopy.add(dest)
1278                 if diff['src_sha1'] != diff['dst_sha1']:
1279                     p4_edit(dest)
1280                     pureRenameCopy.discard(dest)
1281                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1282                     p4_edit(dest)
1283                     pureRenameCopy.discard(dest)
1284                     filesToChangeExecBit[dest] = diff['dst_mode']
1285                 if self.isWindows:
1286                     # turn off read-only attribute
1287                     os.chmod(dest, stat.S_IWRITE)
1288                 os.unlink(dest)
1289                 editedFiles.add(dest)
1290             elif modifier == "R":
1291                 src, dest = diff['src'], diff['dst']
1292                 if self.p4HasMoveCommand:
1293                     p4_edit(src)        # src must be open before move
1294                     p4_move(src, dest)  # opens for (move/delete, move/add)
1295                 else:
1296                     p4_integrate(src, dest)
1297                     if diff['src_sha1'] != diff['dst_sha1']:
1298                         p4_edit(dest)
1299                     else:
1300                         pureRenameCopy.add(dest)
1301                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1302                     if not self.p4HasMoveCommand:
1303                         p4_edit(dest)   # with move: already open, writable
1304                     filesToChangeExecBit[dest] = diff['dst_mode']
1305                 if not self.p4HasMoveCommand:
1306                     if self.isWindows:
1307                         os.chmod(dest, stat.S_IWRITE)
1308                     os.unlink(dest)
1309                     filesToDelete.add(src)
1310                 editedFiles.add(dest)
1311             else:
1312                 die("unknown modifier %s for %s" % (modifier, path))
1313
1314         diffcmd = "git diff-tree -p \"%s\"" % (id)
1315         patchcmd = diffcmd + " | git apply "
1316         tryPatchCmd = patchcmd + "--check -"
1317         applyPatchCmd = patchcmd + "--check --apply -"
1318         patch_succeeded = True
1319
1320         if os.system(tryPatchCmd) != 0:
1321             fixed_rcs_keywords = False
1322             patch_succeeded = False
1323             print "Unfortunately applying the change failed!"
1324
1325             # Patch failed, maybe it's just RCS keyword woes. Look through
1326             # the patch to see if that's possible.
1327             if gitConfigBool("git-p4.attemptRCSCleanup"):
1328                 file = None
1329                 pattern = None
1330                 kwfiles = {}
1331                 for file in editedFiles | filesToDelete:
1332                     # did this file's delta contain RCS keywords?
1333                     pattern = p4_keywords_regexp_for_file(file)
1334
1335                     if pattern:
1336                         # this file is a possibility...look for RCS keywords.
1337                         regexp = re.compile(pattern, re.VERBOSE)
1338                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1339                             if regexp.search(line):
1340                                 if verbose:
1341                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1342                                 kwfiles[file] = pattern
1343                                 break
1344
1345                 for file in kwfiles:
1346                     if verbose:
1347                         print "zapping %s with %s" % (line,pattern)
1348                     # File is being deleted, so not open in p4.  Must
1349                     # disable the read-only bit on windows.
1350                     if self.isWindows and file not in editedFiles:
1351                         os.chmod(file, stat.S_IWRITE)
1352                     self.patchRCSKeywords(file, kwfiles[file])
1353                     fixed_rcs_keywords = True
1354
1355             if fixed_rcs_keywords:
1356                 print "Retrying the patch with RCS keywords cleaned up"
1357                 if os.system(tryPatchCmd) == 0:
1358                     patch_succeeded = True
1359
1360         if not patch_succeeded:
1361             for f in editedFiles:
1362                 p4_revert(f)
1363             return False
1364
1365         #
1366         # Apply the patch for real, and do add/delete/+x handling.
1367         #
1368         system(applyPatchCmd)
1369
1370         for f in filesToAdd:
1371             p4_add(f)
1372         for f in filesToDelete:
1373             p4_revert(f)
1374             p4_delete(f)
1375
1376         # Set/clear executable bits
1377         for f in filesToChangeExecBit.keys():
1378             mode = filesToChangeExecBit[f]
1379             setP4ExecBit(f, mode)
1380
1381         #
1382         # Build p4 change description, starting with the contents
1383         # of the git commit message.
1384         #
1385         logMessage = extractLogMessageFromGitCommit(id)
1386         logMessage = logMessage.strip()
1387         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1388
1389         template = self.prepareSubmitTemplate()
1390         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1391
1392         if self.preserveUser:
1393            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1394
1395         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1396             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1397             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1398             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1399
1400         separatorLine = "######## everything below this line is just the diff #######\n"
1401
1402         # diff
1403         if os.environ.has_key("P4DIFF"):
1404             del(os.environ["P4DIFF"])
1405         diff = ""
1406         for editedFile in editedFiles:
1407             diff += p4_read_pipe(['diff', '-du',
1408                                   wildcard_encode(editedFile)])
1409
1410         # new file diff
1411         newdiff = ""
1412         for newFile in filesToAdd:
1413             newdiff += "==== new file ====\n"
1414             newdiff += "--- /dev/null\n"
1415             newdiff += "+++ %s\n" % newFile
1416             f = open(newFile, "r")
1417             for line in f.readlines():
1418                 newdiff += "+" + line
1419             f.close()
1420
1421         # change description file: submitTemplate, separatorLine, diff, newdiff
1422         (handle, fileName) = tempfile.mkstemp()
1423         tmpFile = os.fdopen(handle, "w+")
1424         if self.isWindows:
1425             submitTemplate = submitTemplate.replace("\n", "\r\n")
1426             separatorLine = separatorLine.replace("\n", "\r\n")
1427             newdiff = newdiff.replace("\n", "\r\n")
1428         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1429         tmpFile.close()
1430
1431         if self.prepare_p4_only:
1432             #
1433             # Leave the p4 tree prepared, and the submit template around
1434             # and let the user decide what to do next
1435             #
1436             print
1437             print "P4 workspace prepared for submission."
1438             print "To submit or revert, go to client workspace"
1439             print "  " + self.clientPath
1440             print
1441             print "To submit, use \"p4 submit\" to write a new description,"
1442             print "or \"p4 submit -i %s\" to use the one prepared by" \
1443                   " \"git p4\"." % fileName
1444             print "You can delete the file \"%s\" when finished." % fileName
1445
1446             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1447                 print "To preserve change ownership by user %s, you must\n" \
1448                       "do \"p4 change -f <change>\" after submitting and\n" \
1449                       "edit the User field."
1450             if pureRenameCopy:
1451                 print "After submitting, renamed files must be re-synced."
1452                 print "Invoke \"p4 sync -f\" on each of these files:"
1453                 for f in pureRenameCopy:
1454                     print "  " + f
1455
1456             print
1457             print "To revert the changes, use \"p4 revert ...\", and delete"
1458             print "the submit template file \"%s\"" % fileName
1459             if filesToAdd:
1460                 print "Since the commit adds new files, they must be deleted:"
1461                 for f in filesToAdd:
1462                     print "  " + f
1463             print
1464             return True
1465
1466         #
1467         # Let the user edit the change description, then submit it.
1468         #
1469         if self.edit_template(fileName):
1470             # read the edited message and submit
1471             ret = True
1472             tmpFile = open(fileName, "rb")
1473             message = tmpFile.read()
1474             tmpFile.close()
1475             submitTemplate = message[:message.index(separatorLine)]
1476             if self.isWindows:
1477                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1478             p4_write_pipe(['submit', '-i'], submitTemplate)
1479
1480             if self.preserveUser:
1481                 if p4User:
1482                     # Get last changelist number. Cannot easily get it from
1483                     # the submit command output as the output is
1484                     # unmarshalled.
1485                     changelist = self.lastP4Changelist()
1486                     self.modifyChangelistUser(changelist, p4User)
1487
1488             # The rename/copy happened by applying a patch that created a
1489             # new file.  This leaves it writable, which confuses p4.
1490             for f in pureRenameCopy:
1491                 p4_sync(f, "-f")
1492
1493         else:
1494             # skip this patch
1495             ret = False
1496             print "Submission cancelled, undoing p4 changes."
1497             for f in editedFiles:
1498                 p4_revert(f)
1499             for f in filesToAdd:
1500                 p4_revert(f)
1501                 os.remove(f)
1502             for f in filesToDelete:
1503                 p4_revert(f)
1504
1505         os.remove(fileName)
1506         return ret
1507
1508     # Export git tags as p4 labels. Create a p4 label and then tag
1509     # with that.
1510     def exportGitTags(self, gitTags):
1511         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1512         if len(validLabelRegexp) == 0:
1513             validLabelRegexp = defaultLabelRegexp
1514         m = re.compile(validLabelRegexp)
1515
1516         for name in gitTags:
1517
1518             if not m.match(name):
1519                 if verbose:
1520                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1521                 continue
1522
1523             # Get the p4 commit this corresponds to
1524             logMessage = extractLogMessageFromGitCommit(name)
1525             values = extractSettingsGitLog(logMessage)
1526
1527             if not values.has_key('change'):
1528                 # a tag pointing to something not sent to p4; ignore
1529                 if verbose:
1530                     print "git tag %s does not give a p4 commit" % name
1531                 continue
1532             else:
1533                 changelist = values['change']
1534
1535             # Get the tag details.
1536             inHeader = True
1537             isAnnotated = False
1538             body = []
1539             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1540                 l = l.strip()
1541                 if inHeader:
1542                     if re.match(r'tag\s+', l):
1543                         isAnnotated = True
1544                     elif re.match(r'\s*$', l):
1545                         inHeader = False
1546                         continue
1547                 else:
1548                     body.append(l)
1549
1550             if not isAnnotated:
1551                 body = ["lightweight tag imported by git p4\n"]
1552
1553             # Create the label - use the same view as the client spec we are using
1554             clientSpec = getClientSpec()
1555
1556             labelTemplate  = "Label: %s\n" % name
1557             labelTemplate += "Description:\n"
1558             for b in body:
1559                 labelTemplate += "\t" + b + "\n"
1560             labelTemplate += "View:\n"
1561             for depot_side in clientSpec.mappings:
1562                 labelTemplate += "\t%s\n" % depot_side
1563
1564             if self.dry_run:
1565                 print "Would create p4 label %s for tag" % name
1566             elif self.prepare_p4_only:
1567                 print "Not creating p4 label %s for tag due to option" \
1568                       " --prepare-p4-only" % name
1569             else:
1570                 p4_write_pipe(["label", "-i"], labelTemplate)
1571
1572                 # Use the label
1573                 p4_system(["tag", "-l", name] +
1574                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1575
1576                 if verbose:
1577                     print "created p4 label for tag %s" % name
1578
1579     def run(self, args):
1580         if len(args) == 0:
1581             self.master = currentGitBranch()
1582             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1583                 die("Detecting current git branch failed!")
1584         elif len(args) == 1:
1585             self.master = args[0]
1586             if not branchExists(self.master):
1587                 die("Branch %s does not exist" % self.master)
1588         else:
1589             return False
1590
1591         allowSubmit = gitConfig("git-p4.allowSubmit")
1592         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1593             die("%s is not in git-p4.allowSubmit" % self.master)
1594
1595         [upstream, settings] = findUpstreamBranchPoint()
1596         self.depotPath = settings['depot-paths'][0]
1597         if len(self.origin) == 0:
1598             self.origin = upstream
1599
1600         if self.preserveUser:
1601             if not self.canChangeChangelists():
1602                 die("Cannot preserve user names without p4 super-user or admin permissions")
1603
1604         # if not set from the command line, try the config file
1605         if self.conflict_behavior is None:
1606             val = gitConfig("git-p4.conflict")
1607             if val:
1608                 if val not in self.conflict_behavior_choices:
1609                     die("Invalid value '%s' for config git-p4.conflict" % val)
1610             else:
1611                 val = "ask"
1612             self.conflict_behavior = val
1613
1614         if self.verbose:
1615             print "Origin branch is " + self.origin
1616
1617         if len(self.depotPath) == 0:
1618             print "Internal error: cannot locate perforce depot path from existing branches"
1619             sys.exit(128)
1620
1621         self.useClientSpec = False
1622         if gitConfigBool("git-p4.useclientspec"):
1623             self.useClientSpec = True
1624         if self.useClientSpec:
1625             self.clientSpecDirs = getClientSpec()
1626
1627         if self.useClientSpec:
1628             # all files are relative to the client spec
1629             self.clientPath = getClientRoot()
1630         else:
1631             self.clientPath = p4Where(self.depotPath)
1632
1633         if self.clientPath == "":
1634             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1635
1636         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1637         self.oldWorkingDirectory = os.getcwd()
1638
1639         # ensure the clientPath exists
1640         new_client_dir = False
1641         if not os.path.exists(self.clientPath):
1642             new_client_dir = True
1643             os.makedirs(self.clientPath)
1644
1645         chdir(self.clientPath, is_client_path=True)
1646         if self.dry_run:
1647             print "Would synchronize p4 checkout in %s" % self.clientPath
1648         else:
1649             print "Synchronizing p4 checkout..."
1650             if new_client_dir:
1651                 # old one was destroyed, and maybe nobody told p4
1652                 p4_sync("...", "-f")
1653             else:
1654                 p4_sync("...")
1655         self.check()
1656
1657         commits = []
1658         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1659             commits.append(line.strip())
1660         commits.reverse()
1661
1662         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1663             self.checkAuthorship = False
1664         else:
1665             self.checkAuthorship = True
1666
1667         if self.preserveUser:
1668             self.checkValidP4Users(commits)
1669
1670         #
1671         # Build up a set of options to be passed to diff when
1672         # submitting each commit to p4.
1673         #
1674         if self.detectRenames:
1675             # command-line -M arg
1676             self.diffOpts = "-M"
1677         else:
1678             # If not explicitly set check the config variable
1679             detectRenames = gitConfig("git-p4.detectRenames")
1680
1681             if detectRenames.lower() == "false" or detectRenames == "":
1682                 self.diffOpts = ""
1683             elif detectRenames.lower() == "true":
1684                 self.diffOpts = "-M"
1685             else:
1686                 self.diffOpts = "-M%s" % detectRenames
1687
1688         # no command-line arg for -C or --find-copies-harder, just
1689         # config variables
1690         detectCopies = gitConfig("git-p4.detectCopies")
1691         if detectCopies.lower() == "false" or detectCopies == "":
1692             pass
1693         elif detectCopies.lower() == "true":
1694             self.diffOpts += " -C"
1695         else:
1696             self.diffOpts += " -C%s" % detectCopies
1697
1698         if gitConfigBool("git-p4.detectCopiesHarder"):
1699             self.diffOpts += " --find-copies-harder"
1700
1701         #
1702         # Apply the commits, one at a time.  On failure, ask if should
1703         # continue to try the rest of the patches, or quit.
1704         #
1705         if self.dry_run:
1706             print "Would apply"
1707         applied = []
1708         last = len(commits) - 1
1709         for i, commit in enumerate(commits):
1710             if self.dry_run:
1711                 print " ", read_pipe(["git", "show", "-s",
1712                                       "--format=format:%h %s", commit])
1713                 ok = True
1714             else:
1715                 ok = self.applyCommit(commit)
1716             if ok:
1717                 applied.append(commit)
1718             else:
1719                 if self.prepare_p4_only and i < last:
1720                     print "Processing only the first commit due to option" \
1721                           " --prepare-p4-only"
1722                     break
1723                 if i < last:
1724                     quit = False
1725                     while True:
1726                         # prompt for what to do, or use the option/variable
1727                         if self.conflict_behavior == "ask":
1728                             print "What do you want to do?"
1729                             response = raw_input("[s]kip this commit but apply"
1730                                                  " the rest, or [q]uit? ")
1731                             if not response:
1732                                 continue
1733                         elif self.conflict_behavior == "skip":
1734                             response = "s"
1735                         elif self.conflict_behavior == "quit":
1736                             response = "q"
1737                         else:
1738                             die("Unknown conflict_behavior '%s'" %
1739                                 self.conflict_behavior)
1740
1741                         if response[0] == "s":
1742                             print "Skipping this commit, but applying the rest"
1743                             break
1744                         if response[0] == "q":
1745                             print "Quitting"
1746                             quit = True
1747                             break
1748                     if quit:
1749                         break
1750
1751         chdir(self.oldWorkingDirectory)
1752
1753         if self.dry_run:
1754             pass
1755         elif self.prepare_p4_only:
1756             pass
1757         elif len(commits) == len(applied):
1758             print "All commits applied!"
1759
1760             sync = P4Sync()
1761             if self.branch:
1762                 sync.branch = self.branch
1763             sync.run([])
1764
1765             rebase = P4Rebase()
1766             rebase.rebase()
1767
1768         else:
1769             if len(applied) == 0:
1770                 print "No commits applied."
1771             else:
1772                 print "Applied only the commits marked with '*':"
1773                 for c in commits:
1774                     if c in applied:
1775                         star = "*"
1776                     else:
1777                         star = " "
1778                     print star, read_pipe(["git", "show", "-s",
1779                                            "--format=format:%h %s",  c])
1780                 print "You will have to do 'git p4 sync' and rebase."
1781
1782         if gitConfigBool("git-p4.exportLabels"):
1783             self.exportLabels = True
1784
1785         if self.exportLabels:
1786             p4Labels = getP4Labels(self.depotPath)
1787             gitTags = getGitTags()
1788
1789             missingGitTags = gitTags - p4Labels
1790             self.exportGitTags(missingGitTags)
1791
1792         # exit with error unless everything applied perfectly
1793         if len(commits) != len(applied):
1794                 sys.exit(1)
1795
1796         return True
1797
1798 class View(object):
1799     """Represent a p4 view ("p4 help views"), and map files in a
1800        repo according to the view."""
1801
1802     def __init__(self, client_name):
1803         self.mappings = []
1804         self.client_prefix = "//%s/" % client_name
1805         # cache results of "p4 where" to lookup client file locations
1806         self.client_spec_path_cache = {}
1807
1808     def append(self, view_line):
1809         """Parse a view line, splitting it into depot and client
1810            sides.  Append to self.mappings, preserving order.  This
1811            is only needed for tag creation."""
1812
1813         # Split the view line into exactly two words.  P4 enforces
1814         # structure on these lines that simplifies this quite a bit.
1815         #
1816         # Either or both words may be double-quoted.
1817         # Single quotes do not matter.
1818         # Double-quote marks cannot occur inside the words.
1819         # A + or - prefix is also inside the quotes.
1820         # There are no quotes unless they contain a space.
1821         # The line is already white-space stripped.
1822         # The two words are separated by a single space.
1823         #
1824         if view_line[0] == '"':
1825             # First word is double quoted.  Find its end.
1826             close_quote_index = view_line.find('"', 1)
1827             if close_quote_index <= 0:
1828                 die("No first-word closing quote found: %s" % view_line)
1829             depot_side = view_line[1:close_quote_index]
1830             # skip closing quote and space
1831             rhs_index = close_quote_index + 1 + 1
1832         else:
1833             space_index = view_line.find(" ")
1834             if space_index <= 0:
1835                 die("No word-splitting space found: %s" % view_line)
1836             depot_side = view_line[0:space_index]
1837             rhs_index = space_index + 1
1838
1839         # prefix + means overlay on previous mapping
1840         if depot_side.startswith("+"):
1841             depot_side = depot_side[1:]
1842
1843         # prefix - means exclude this path, leave out of mappings
1844         exclude = False
1845         if depot_side.startswith("-"):
1846             exclude = True
1847             depot_side = depot_side[1:]
1848
1849         if not exclude:
1850             self.mappings.append(depot_side)
1851
1852     def convert_client_path(self, clientFile):
1853         # chop off //client/ part to make it relative
1854         if not clientFile.startswith(self.client_prefix):
1855             die("No prefix '%s' on clientFile '%s'" %
1856                 (self.client_prefix, clientFile))
1857         return clientFile[len(self.client_prefix):]
1858
1859     def update_client_spec_path_cache(self, files):
1860         """ Caching file paths by "p4 where" batch query """
1861
1862         # List depot file paths exclude that already cached
1863         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1864
1865         if len(fileArgs) == 0:
1866             return  # All files in cache
1867
1868         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1869         for res in where_result:
1870             if "code" in res and res["code"] == "error":
1871                 # assume error is "... file(s) not in client view"
1872                 continue
1873             if "clientFile" not in res:
1874                 die("No clientFile from 'p4 where %s'" % depot_path)
1875             if "unmap" in res:
1876                 # it will list all of them, but only one not unmap-ped
1877                 continue
1878             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1879
1880         # not found files or unmap files set to ""
1881         for depotFile in fileArgs:
1882             if depotFile not in self.client_spec_path_cache:
1883                 self.client_spec_path_cache[depotFile] = ""
1884
1885     def map_in_client(self, depot_path):
1886         """Return the relative location in the client where this
1887            depot file should live.  Returns "" if the file should
1888            not be mapped in the client."""
1889
1890         if depot_path in self.client_spec_path_cache:
1891             return self.client_spec_path_cache[depot_path]
1892
1893         die( "Error: %s is not found in client spec path" % depot_path )
1894         return ""
1895
1896 class P4Sync(Command, P4UserMap):
1897     delete_actions = ( "delete", "move/delete", "purge" )
1898
1899     def __init__(self):
1900         Command.__init__(self)
1901         P4UserMap.__init__(self)
1902         self.options = [
1903                 optparse.make_option("--branch", dest="branch"),
1904                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1905                 optparse.make_option("--changesfile", dest="changesFile"),
1906                 optparse.make_option("--silent", dest="silent", action="store_true"),
1907                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1908                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1909                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1910                                      help="Import into refs/heads/ , not refs/remotes"),
1911                 optparse.make_option("--max-changes", dest="maxChanges"),
1912                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1913                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1914                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1915                                      help="Only sync files that are included in the Perforce Client Spec")
1916         ]
1917         self.description = """Imports from Perforce into a git repository.\n
1918     example:
1919     //depot/my/project/ -- to import the current head
1920     //depot/my/project/@all -- to import everything
1921     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1922
1923     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1924
1925         self.usage += " //depot/path[@revRange]"
1926         self.silent = False
1927         self.createdBranches = set()
1928         self.committedChanges = set()
1929         self.branch = ""
1930         self.detectBranches = False
1931         self.detectLabels = False
1932         self.importLabels = False
1933         self.changesFile = ""
1934         self.syncWithOrigin = True
1935         self.importIntoRemotes = True
1936         self.maxChanges = ""
1937         self.keepRepoPath = False
1938         self.depotPaths = None
1939         self.p4BranchesInGit = []
1940         self.cloneExclude = []
1941         self.useClientSpec = False
1942         self.useClientSpec_from_options = False
1943         self.clientSpecDirs = None
1944         self.tempBranches = []
1945         self.tempBranchLocation = "git-p4-tmp"
1946
1947         if gitConfig("git-p4.syncFromOrigin") == "false":
1948             self.syncWithOrigin = False
1949
1950     # Force a checkpoint in fast-import and wait for it to finish
1951     def checkpoint(self):
1952         self.gitStream.write("checkpoint\n\n")
1953         self.gitStream.write("progress checkpoint\n\n")
1954         out = self.gitOutput.readline()
1955         if self.verbose:
1956             print "checkpoint finished: " + out
1957
1958     def extractFilesFromCommit(self, commit):
1959         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1960                              for path in self.cloneExclude]
1961         files = []
1962         fnum = 0
1963         while commit.has_key("depotFile%s" % fnum):
1964             path =  commit["depotFile%s" % fnum]
1965
1966             if [p for p in self.cloneExclude
1967                 if p4PathStartsWith(path, p)]:
1968                 found = False
1969             else:
1970                 found = [p for p in self.depotPaths
1971                          if p4PathStartsWith(path, p)]
1972             if not found:
1973                 fnum = fnum + 1
1974                 continue
1975
1976             file = {}
1977             file["path"] = path
1978             file["rev"] = commit["rev%s" % fnum]
1979             file["action"] = commit["action%s" % fnum]
1980             file["type"] = commit["type%s" % fnum]
1981             files.append(file)
1982             fnum = fnum + 1
1983         return files
1984
1985     def stripRepoPath(self, path, prefixes):
1986         """When streaming files, this is called to map a p4 depot path
1987            to where it should go in git.  The prefixes are either
1988            self.depotPaths, or self.branchPrefixes in the case of
1989            branch detection."""
1990
1991         if self.useClientSpec:
1992             # branch detection moves files up a level (the branch name)
1993             # from what client spec interpretation gives
1994             path = self.clientSpecDirs.map_in_client(path)
1995             if self.detectBranches:
1996                 for b in self.knownBranches:
1997                     if path.startswith(b + "/"):
1998                         path = path[len(b)+1:]
1999
2000         elif self.keepRepoPath:
2001             # Preserve everything in relative path name except leading
2002             # //depot/; just look at first prefix as they all should
2003             # be in the same depot.
2004             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2005             if p4PathStartsWith(path, depot):
2006                 path = path[len(depot):]
2007
2008         else:
2009             for p in prefixes:
2010                 if p4PathStartsWith(path, p):
2011                     path = path[len(p):]
2012                     break
2013
2014         path = wildcard_decode(path)
2015         return path
2016
2017     def splitFilesIntoBranches(self, commit):
2018         """Look at each depotFile in the commit to figure out to what
2019            branch it belongs."""
2020
2021         if self.clientSpecDirs:
2022             files = self.extractFilesFromCommit(commit)
2023             self.clientSpecDirs.update_client_spec_path_cache(files)
2024
2025         branches = {}
2026         fnum = 0
2027         while commit.has_key("depotFile%s" % fnum):
2028             path =  commit["depotFile%s" % fnum]
2029             found = [p for p in self.depotPaths
2030                      if p4PathStartsWith(path, p)]
2031             if not found:
2032                 fnum = fnum + 1
2033                 continue
2034
2035             file = {}
2036             file["path"] = path
2037             file["rev"] = commit["rev%s" % fnum]
2038             file["action"] = commit["action%s" % fnum]
2039             file["type"] = commit["type%s" % fnum]
2040             fnum = fnum + 1
2041
2042             # start with the full relative path where this file would
2043             # go in a p4 client
2044             if self.useClientSpec:
2045                 relPath = self.clientSpecDirs.map_in_client(path)
2046             else:
2047                 relPath = self.stripRepoPath(path, self.depotPaths)
2048
2049             for branch in self.knownBranches.keys():
2050                 # add a trailing slash so that a commit into qt/4.2foo
2051                 # doesn't end up in qt/4.2, e.g.
2052                 if relPath.startswith(branch + "/"):
2053                     if branch not in branches:
2054                         branches[branch] = []
2055                     branches[branch].append(file)
2056                     break
2057
2058         return branches
2059
2060     # output one file from the P4 stream
2061     # - helper for streamP4Files
2062
2063     def streamOneP4File(self, file, contents):
2064         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2065         if verbose:
2066             sys.stderr.write("%s\n" % relPath)
2067
2068         (type_base, type_mods) = split_p4_type(file["type"])
2069
2070         git_mode = "100644"
2071         if "x" in type_mods:
2072             git_mode = "100755"
2073         if type_base == "symlink":
2074             git_mode = "120000"
2075             # p4 print on a symlink sometimes contains "target\n";
2076             # if it does, remove the newline
2077             data = ''.join(contents)
2078             if data[-1] == '\n':
2079                 contents = [data[:-1]]
2080             else:
2081                 contents = [data]
2082
2083         if type_base == "utf16":
2084             # p4 delivers different text in the python output to -G
2085             # than it does when using "print -o", or normal p4 client
2086             # operations.  utf16 is converted to ascii or utf8, perhaps.
2087             # But ascii text saved as -t utf16 is completely mangled.
2088             # Invoke print -o to get the real contents.
2089             #
2090             # On windows, the newlines will always be mangled by print, so put
2091             # them back too.  This is not needed to the cygwin windows version,
2092             # just the native "NT" type.
2093             #
2094             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2095             if p4_version_string().find("/NT") >= 0:
2096                 text = text.replace("\r\n", "\n")
2097             contents = [ text ]
2098
2099         if type_base == "apple":
2100             # Apple filetype files will be streamed as a concatenation of
2101             # its appledouble header and the contents.  This is useless
2102             # on both macs and non-macs.  If using "print -q -o xx", it
2103             # will create "xx" with the data, and "%xx" with the header.
2104             # This is also not very useful.
2105             #
2106             # Ideally, someday, this script can learn how to generate
2107             # appledouble files directly and import those to git, but
2108             # non-mac machines can never find a use for apple filetype.
2109             print "\nIgnoring apple filetype file %s" % file['depotFile']
2110             return
2111
2112         # Note that we do not try to de-mangle keywords on utf16 files,
2113         # even though in theory somebody may want that.
2114         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2115         if pattern:
2116             regexp = re.compile(pattern, re.VERBOSE)
2117             text = ''.join(contents)
2118             text = regexp.sub(r'$\1$', text)
2119             contents = [ text ]
2120
2121         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2122
2123         # total length...
2124         length = 0
2125         for d in contents:
2126             length = length + len(d)
2127
2128         self.gitStream.write("data %d\n" % length)
2129         for d in contents:
2130             self.gitStream.write(d)
2131         self.gitStream.write("\n")
2132
2133     def streamOneP4Deletion(self, file):
2134         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2135         if verbose:
2136             sys.stderr.write("delete %s\n" % relPath)
2137         self.gitStream.write("D %s\n" % relPath)
2138
2139     # handle another chunk of streaming data
2140     def streamP4FilesCb(self, marshalled):
2141
2142         # catch p4 errors and complain
2143         err = None
2144         if "code" in marshalled:
2145             if marshalled["code"] == "error":
2146                 if "data" in marshalled:
2147                     err = marshalled["data"].rstrip()
2148         if err:
2149             f = None
2150             if self.stream_have_file_info:
2151                 if "depotFile" in self.stream_file:
2152                     f = self.stream_file["depotFile"]
2153             # force a failure in fast-import, else an empty
2154             # commit will be made
2155             self.gitStream.write("\n")
2156             self.gitStream.write("die-now\n")
2157             self.gitStream.close()
2158             # ignore errors, but make sure it exits first
2159             self.importProcess.wait()
2160             if f:
2161                 die("Error from p4 print for %s: %s" % (f, err))
2162             else:
2163                 die("Error from p4 print: %s" % err)
2164
2165         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2166             # start of a new file - output the old one first
2167             self.streamOneP4File(self.stream_file, self.stream_contents)
2168             self.stream_file = {}
2169             self.stream_contents = []
2170             self.stream_have_file_info = False
2171
2172         # pick up the new file information... for the
2173         # 'data' field we need to append to our array
2174         for k in marshalled.keys():
2175             if k == 'data':
2176                 self.stream_contents.append(marshalled['data'])
2177             else:
2178                 self.stream_file[k] = marshalled[k]
2179
2180         self.stream_have_file_info = True
2181
2182     # Stream directly from "p4 files" into "git fast-import"
2183     def streamP4Files(self, files):
2184         filesForCommit = []
2185         filesToRead = []
2186         filesToDelete = []
2187
2188         for f in files:
2189             # if using a client spec, only add the files that have
2190             # a path in the client
2191             if self.clientSpecDirs:
2192                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2193                     continue
2194
2195             filesForCommit.append(f)
2196             if f['action'] in self.delete_actions:
2197                 filesToDelete.append(f)
2198             else:
2199                 filesToRead.append(f)
2200
2201         # deleted files...
2202         for f in filesToDelete:
2203             self.streamOneP4Deletion(f)
2204
2205         if len(filesToRead) > 0:
2206             self.stream_file = {}
2207             self.stream_contents = []
2208             self.stream_have_file_info = False
2209
2210             # curry self argument
2211             def streamP4FilesCbSelf(entry):
2212                 self.streamP4FilesCb(entry)
2213
2214             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2215
2216             p4CmdList(["-x", "-", "print"],
2217                       stdin=fileArgs,
2218                       cb=streamP4FilesCbSelf)
2219
2220             # do the last chunk
2221             if self.stream_file.has_key('depotFile'):
2222                 self.streamOneP4File(self.stream_file, self.stream_contents)
2223
2224     def make_email(self, userid):
2225         if userid in self.users:
2226             return self.users[userid]
2227         else:
2228             return "%s <a@b>" % userid
2229
2230     # Stream a p4 tag
2231     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2232         if verbose:
2233             print "writing tag %s for commit %s" % (labelName, commit)
2234         gitStream.write("tag %s\n" % labelName)
2235         gitStream.write("from %s\n" % commit)
2236
2237         if labelDetails.has_key('Owner'):
2238             owner = labelDetails["Owner"]
2239         else:
2240             owner = None
2241
2242         # Try to use the owner of the p4 label, or failing that,
2243         # the current p4 user id.
2244         if owner:
2245             email = self.make_email(owner)
2246         else:
2247             email = self.make_email(self.p4UserId())
2248         tagger = "%s %s %s" % (email, epoch, self.tz)
2249
2250         gitStream.write("tagger %s\n" % tagger)
2251
2252         print "labelDetails=",labelDetails
2253         if labelDetails.has_key('Description'):
2254             description = labelDetails['Description']
2255         else:
2256             description = 'Label from git p4'
2257
2258         gitStream.write("data %d\n" % len(description))
2259         gitStream.write(description)
2260         gitStream.write("\n")
2261
2262     def commit(self, details, files, branch, parent = ""):
2263         epoch = details["time"]
2264         author = details["user"]
2265
2266         if self.verbose:
2267             print "commit into %s" % branch
2268
2269         # start with reading files; if that fails, we should not
2270         # create a commit.
2271         new_files = []
2272         for f in files:
2273             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2274                 new_files.append (f)
2275             else:
2276                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2277
2278         if self.clientSpecDirs:
2279             self.clientSpecDirs.update_client_spec_path_cache(files)
2280
2281         self.gitStream.write("commit %s\n" % branch)
2282 #        gitStream.write("mark :%s\n" % details["change"])
2283         self.committedChanges.add(int(details["change"]))
2284         committer = ""
2285         if author not in self.users:
2286             self.getUserMapFromPerforceServer()
2287         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2288
2289         self.gitStream.write("committer %s\n" % committer)
2290
2291         self.gitStream.write("data <<EOT\n")
2292         self.gitStream.write(details["desc"])
2293         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2294                              (','.join(self.branchPrefixes), details["change"]))
2295         if len(details['options']) > 0:
2296             self.gitStream.write(": options = %s" % details['options'])
2297         self.gitStream.write("]\nEOT\n\n")
2298
2299         if len(parent) > 0:
2300             if self.verbose:
2301                 print "parent %s" % parent
2302             self.gitStream.write("from %s\n" % parent)
2303
2304         self.streamP4Files(new_files)
2305         self.gitStream.write("\n")
2306
2307         change = int(details["change"])
2308
2309         if self.labels.has_key(change):
2310             label = self.labels[change]
2311             labelDetails = label[0]
2312             labelRevisions = label[1]
2313             if self.verbose:
2314                 print "Change %s is labelled %s" % (change, labelDetails)
2315
2316             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2317                                                 for p in self.branchPrefixes])
2318
2319             if len(files) == len(labelRevisions):
2320
2321                 cleanedFiles = {}
2322                 for info in files:
2323                     if info["action"] in self.delete_actions:
2324                         continue
2325                     cleanedFiles[info["depotFile"]] = info["rev"]
2326
2327                 if cleanedFiles == labelRevisions:
2328                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2329
2330                 else:
2331                     if not self.silent:
2332                         print ("Tag %s does not match with change %s: files do not match."
2333                                % (labelDetails["label"], change))
2334
2335             else:
2336                 if not self.silent:
2337                     print ("Tag %s does not match with change %s: file count is different."
2338                            % (labelDetails["label"], change))
2339
2340     # Build a dictionary of changelists and labels, for "detect-labels" option.
2341     def getLabels(self):
2342         self.labels = {}
2343
2344         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2345         if len(l) > 0 and not self.silent:
2346             print "Finding files belonging to labels in %s" % `self.depotPaths`
2347
2348         for output in l:
2349             label = output["label"]
2350             revisions = {}
2351             newestChange = 0
2352             if self.verbose:
2353                 print "Querying files for label %s" % label
2354             for file in p4CmdList(["files"] +
2355                                       ["%s...@%s" % (p, label)
2356                                           for p in self.depotPaths]):
2357                 revisions[file["depotFile"]] = file["rev"]
2358                 change = int(file["change"])
2359                 if change > newestChange:
2360                     newestChange = change
2361
2362             self.labels[newestChange] = [output, revisions]
2363
2364         if self.verbose:
2365             print "Label changes: %s" % self.labels.keys()
2366
2367     # Import p4 labels as git tags. A direct mapping does not
2368     # exist, so assume that if all the files are at the same revision
2369     # then we can use that, or it's something more complicated we should
2370     # just ignore.
2371     def importP4Labels(self, stream, p4Labels):
2372         if verbose:
2373             print "import p4 labels: " + ' '.join(p4Labels)
2374
2375         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2376         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2377         if len(validLabelRegexp) == 0:
2378             validLabelRegexp = defaultLabelRegexp
2379         m = re.compile(validLabelRegexp)
2380
2381         for name in p4Labels:
2382             commitFound = False
2383
2384             if not m.match(name):
2385                 if verbose:
2386                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2387                 continue
2388
2389             if name in ignoredP4Labels:
2390                 continue
2391
2392             labelDetails = p4CmdList(['label', "-o", name])[0]
2393
2394             # get the most recent changelist for each file in this label
2395             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2396                                 for p in self.depotPaths])
2397
2398             if change.has_key('change'):
2399                 # find the corresponding git commit; take the oldest commit
2400                 changelist = int(change['change'])
2401                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2402                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2403                 if len(gitCommit) == 0:
2404                     print "could not find git commit for changelist %d" % changelist
2405                 else:
2406                     gitCommit = gitCommit.strip()
2407                     commitFound = True
2408                     # Convert from p4 time format
2409                     try:
2410                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2411                     except ValueError:
2412                         print "Could not convert label time %s" % labelDetails['Update']
2413                         tmwhen = 1
2414
2415                     when = int(time.mktime(tmwhen))
2416                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2417                     if verbose:
2418                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2419             else:
2420                 if verbose:
2421                     print "Label %s has no changelists - possibly deleted?" % name
2422
2423             if not commitFound:
2424                 # We can't import this label; don't try again as it will get very
2425                 # expensive repeatedly fetching all the files for labels that will
2426                 # never be imported. If the label is moved in the future, the
2427                 # ignore will need to be removed manually.
2428                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2429
2430     def guessProjectName(self):
2431         for p in self.depotPaths:
2432             if p.endswith("/"):
2433                 p = p[:-1]
2434             p = p[p.strip().rfind("/") + 1:]
2435             if not p.endswith("/"):
2436                p += "/"
2437             return p
2438
2439     def getBranchMapping(self):
2440         lostAndFoundBranches = set()
2441
2442         user = gitConfig("git-p4.branchUser")
2443         if len(user) > 0:
2444             command = "branches -u %s" % user
2445         else:
2446             command = "branches"
2447
2448         for info in p4CmdList(command):
2449             details = p4Cmd(["branch", "-o", info["branch"]])
2450             viewIdx = 0
2451             while details.has_key("View%s" % viewIdx):
2452                 paths = details["View%s" % viewIdx].split(" ")
2453                 viewIdx = viewIdx + 1
2454                 # require standard //depot/foo/... //depot/bar/... mapping
2455                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2456                     continue
2457                 source = paths[0]
2458                 destination = paths[1]
2459                 ## HACK
2460                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2461                     source = source[len(self.depotPaths[0]):-4]
2462                     destination = destination[len(self.depotPaths[0]):-4]
2463
2464                     if destination in self.knownBranches:
2465                         if not self.silent:
2466                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2467                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2468                         continue
2469
2470                     self.knownBranches[destination] = source
2471
2472                     lostAndFoundBranches.discard(destination)
2473
2474                     if source not in self.knownBranches:
2475                         lostAndFoundBranches.add(source)
2476
2477         # Perforce does not strictly require branches to be defined, so we also
2478         # check git config for a branch list.
2479         #
2480         # Example of branch definition in git config file:
2481         # [git-p4]
2482         #   branchList=main:branchA
2483         #   branchList=main:branchB
2484         #   branchList=branchA:branchC
2485         configBranches = gitConfigList("git-p4.branchList")
2486         for branch in configBranches:
2487             if branch:
2488                 (source, destination) = branch.split(":")
2489                 self.knownBranches[destination] = source
2490
2491                 lostAndFoundBranches.discard(destination)
2492
2493                 if source not in self.knownBranches:
2494                     lostAndFoundBranches.add(source)
2495
2496
2497         for branch in lostAndFoundBranches:
2498             self.knownBranches[branch] = branch
2499
2500     def getBranchMappingFromGitBranches(self):
2501         branches = p4BranchesInGit(self.importIntoRemotes)
2502         for branch in branches.keys():
2503             if branch == "master":
2504                 branch = "main"
2505             else:
2506                 branch = branch[len(self.projectName):]
2507             self.knownBranches[branch] = branch
2508
2509     def updateOptionDict(self, d):
2510         option_keys = {}
2511         if self.keepRepoPath:
2512             option_keys['keepRepoPath'] = 1
2513
2514         d["options"] = ' '.join(sorted(option_keys.keys()))
2515
2516     def readOptions(self, d):
2517         self.keepRepoPath = (d.has_key('options')
2518                              and ('keepRepoPath' in d['options']))
2519
2520     def gitRefForBranch(self, branch):
2521         if branch == "main":
2522             return self.refPrefix + "master"
2523
2524         if len(branch) <= 0:
2525             return branch
2526
2527         return self.refPrefix + self.projectName + branch
2528
2529     def gitCommitByP4Change(self, ref, change):
2530         if self.verbose:
2531             print "looking in ref " + ref + " for change %s using bisect..." % change
2532
2533         earliestCommit = ""
2534         latestCommit = parseRevision(ref)
2535
2536         while True:
2537             if self.verbose:
2538                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2539             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2540             if len(next) == 0:
2541                 if self.verbose:
2542                     print "argh"
2543                 return ""
2544             log = extractLogMessageFromGitCommit(next)
2545             settings = extractSettingsGitLog(log)
2546             currentChange = int(settings['change'])
2547             if self.verbose:
2548                 print "current change %s" % currentChange
2549
2550             if currentChange == change:
2551                 if self.verbose:
2552                     print "found %s" % next
2553                 return next
2554
2555             if currentChange < change:
2556                 earliestCommit = "^%s" % next
2557             else:
2558                 latestCommit = "%s" % next
2559
2560         return ""
2561
2562     def importNewBranch(self, branch, maxChange):
2563         # make fast-import flush all changes to disk and update the refs using the checkpoint
2564         # command so that we can try to find the branch parent in the git history
2565         self.gitStream.write("checkpoint\n\n");
2566         self.gitStream.flush();
2567         branchPrefix = self.depotPaths[0] + branch + "/"
2568         range = "@1,%s" % maxChange
2569         #print "prefix" + branchPrefix
2570         changes = p4ChangesForPaths([branchPrefix], range)
2571         if len(changes) <= 0:
2572             return False
2573         firstChange = changes[0]
2574         #print "first change in branch: %s" % firstChange
2575         sourceBranch = self.knownBranches[branch]
2576         sourceDepotPath = self.depotPaths[0] + sourceBranch
2577         sourceRef = self.gitRefForBranch(sourceBranch)
2578         #print "source " + sourceBranch
2579
2580         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2581         #print "branch parent: %s" % branchParentChange
2582         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2583         if len(gitParent) > 0:
2584             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2585             #print "parent git commit: %s" % gitParent
2586
2587         self.importChanges(changes)
2588         return True
2589
2590     def searchParent(self, parent, branch, target):
2591         parentFound = False
2592         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2593                                      "--no-merges", parent]):
2594             blob = blob.strip()
2595             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2596                 parentFound = True
2597                 if self.verbose:
2598                     print "Found parent of %s in commit %s" % (branch, blob)
2599                 break
2600         if parentFound:
2601             return blob
2602         else:
2603             return None
2604
2605     def importChanges(self, changes):
2606         cnt = 1
2607         for change in changes:
2608             description = p4_describe(change)
2609             self.updateOptionDict(description)
2610
2611             if not self.silent:
2612                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2613                 sys.stdout.flush()
2614             cnt = cnt + 1
2615
2616             try:
2617                 if self.detectBranches:
2618                     branches = self.splitFilesIntoBranches(description)
2619                     for branch in branches.keys():
2620                         ## HACK  --hwn
2621                         branchPrefix = self.depotPaths[0] + branch + "/"
2622                         self.branchPrefixes = [ branchPrefix ]
2623
2624                         parent = ""
2625
2626                         filesForCommit = branches[branch]
2627
2628                         if self.verbose:
2629                             print "branch is %s" % branch
2630
2631                         self.updatedBranches.add(branch)
2632
2633                         if branch not in self.createdBranches:
2634                             self.createdBranches.add(branch)
2635                             parent = self.knownBranches[branch]
2636                             if parent == branch:
2637                                 parent = ""
2638                             else:
2639                                 fullBranch = self.projectName + branch
2640                                 if fullBranch not in self.p4BranchesInGit:
2641                                     if not self.silent:
2642                                         print("\n    Importing new branch %s" % fullBranch);
2643                                     if self.importNewBranch(branch, change - 1):
2644                                         parent = ""
2645                                         self.p4BranchesInGit.append(fullBranch)
2646                                     if not self.silent:
2647                                         print("\n    Resuming with change %s" % change);
2648
2649                                 if self.verbose:
2650                                     print "parent determined through known branches: %s" % parent
2651
2652                         branch = self.gitRefForBranch(branch)
2653                         parent = self.gitRefForBranch(parent)
2654
2655                         if self.verbose:
2656                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2657
2658                         if len(parent) == 0 and branch in self.initialParents:
2659                             parent = self.initialParents[branch]
2660                             del self.initialParents[branch]
2661
2662                         blob = None
2663                         if len(parent) > 0:
2664                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2665                             if self.verbose:
2666                                 print "Creating temporary branch: " + tempBranch
2667                             self.commit(description, filesForCommit, tempBranch)
2668                             self.tempBranches.append(tempBranch)
2669                             self.checkpoint()
2670                             blob = self.searchParent(parent, branch, tempBranch)
2671                         if blob:
2672                             self.commit(description, filesForCommit, branch, blob)
2673                         else:
2674                             if self.verbose:
2675                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2676                             self.commit(description, filesForCommit, branch, parent)
2677                 else:
2678                     files = self.extractFilesFromCommit(description)
2679                     self.commit(description, files, self.branch,
2680                                 self.initialParent)
2681                     # only needed once, to connect to the previous commit
2682                     self.initialParent = ""
2683             except IOError:
2684                 print self.gitError.read()
2685                 sys.exit(1)
2686
2687     def importHeadRevision(self, revision):
2688         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2689
2690         details = {}
2691         details["user"] = "git perforce import user"
2692         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2693                            % (' '.join(self.depotPaths), revision))
2694         details["change"] = revision
2695         newestRevision = 0
2696
2697         fileCnt = 0
2698         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2699
2700         for info in p4CmdList(["files"] + fileArgs):
2701
2702             if 'code' in info and info['code'] == 'error':
2703                 sys.stderr.write("p4 returned an error: %s\n"
2704                                  % info['data'])
2705                 if info['data'].find("must refer to client") >= 0:
2706                     sys.stderr.write("This particular p4 error is misleading.\n")
2707                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2708                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2709                 sys.exit(1)
2710             if 'p4ExitCode' in info:
2711                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2712                 sys.exit(1)
2713
2714
2715             change = int(info["change"])
2716             if change > newestRevision:
2717                 newestRevision = change
2718
2719             if info["action"] in self.delete_actions:
2720                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2721                 #fileCnt = fileCnt + 1
2722                 continue
2723
2724             for prop in ["depotFile", "rev", "action", "type" ]:
2725                 details["%s%s" % (prop, fileCnt)] = info[prop]
2726
2727             fileCnt = fileCnt + 1
2728
2729         details["change"] = newestRevision
2730
2731         # Use time from top-most change so that all git p4 clones of
2732         # the same p4 repo have the same commit SHA1s.
2733         res = p4_describe(newestRevision)
2734         details["time"] = res["time"]
2735
2736         self.updateOptionDict(details)
2737         try:
2738             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2739         except IOError:
2740             print "IO error with git fast-import. Is your git version recent enough?"
2741             print self.gitError.read()
2742
2743
2744     def run(self, args):
2745         self.depotPaths = []
2746         self.changeRange = ""
2747         self.previousDepotPaths = []
2748         self.hasOrigin = False
2749
2750         # map from branch depot path to parent branch
2751         self.knownBranches = {}
2752         self.initialParents = {}
2753
2754         if self.importIntoRemotes:
2755             self.refPrefix = "refs/remotes/p4/"
2756         else:
2757             self.refPrefix = "refs/heads/p4/"
2758
2759         if self.syncWithOrigin:
2760             self.hasOrigin = originP4BranchesExist()
2761             if self.hasOrigin:
2762                 if not self.silent:
2763                     print 'Syncing with origin first, using "git fetch origin"'
2764                 system("git fetch origin")
2765
2766         branch_arg_given = bool(self.branch)
2767         if len(self.branch) == 0:
2768             self.branch = self.refPrefix + "master"
2769             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2770                 system("git update-ref %s refs/heads/p4" % self.branch)
2771                 system("git branch -D p4")
2772
2773         # accept either the command-line option, or the configuration variable
2774         if self.useClientSpec:
2775             # will use this after clone to set the variable
2776             self.useClientSpec_from_options = True
2777         else:
2778             if gitConfigBool("git-p4.useclientspec"):
2779                 self.useClientSpec = True
2780         if self.useClientSpec:
2781             self.clientSpecDirs = getClientSpec()
2782
2783         # TODO: should always look at previous commits,
2784         # merge with previous imports, if possible.
2785         if args == []:
2786             if self.hasOrigin:
2787                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2788
2789             # branches holds mapping from branch name to sha1
2790             branches = p4BranchesInGit(self.importIntoRemotes)
2791
2792             # restrict to just this one, disabling detect-branches
2793             if branch_arg_given:
2794                 short = self.branch.split("/")[-1]
2795                 if short in branches:
2796                     self.p4BranchesInGit = [ short ]
2797             else:
2798                 self.p4BranchesInGit = branches.keys()
2799
2800             if len(self.p4BranchesInGit) > 1:
2801                 if not self.silent:
2802                     print "Importing from/into multiple branches"
2803                 self.detectBranches = True
2804                 for branch in branches.keys():
2805                     self.initialParents[self.refPrefix + branch] = \
2806                         branches[branch]
2807
2808             if self.verbose:
2809                 print "branches: %s" % self.p4BranchesInGit
2810
2811             p4Change = 0
2812             for branch in self.p4BranchesInGit:
2813                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2814
2815                 settings = extractSettingsGitLog(logMsg)
2816
2817                 self.readOptions(settings)
2818                 if (settings.has_key('depot-paths')
2819                     and settings.has_key ('change')):
2820                     change = int(settings['change']) + 1
2821                     p4Change = max(p4Change, change)
2822
2823                     depotPaths = sorted(settings['depot-paths'])
2824                     if self.previousDepotPaths == []:
2825                         self.previousDepotPaths = depotPaths
2826                     else:
2827                         paths = []
2828                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2829                             prev_list = prev.split("/")
2830                             cur_list = cur.split("/")
2831                             for i in range(0, min(len(cur_list), len(prev_list))):
2832                                 if cur_list[i] <> prev_list[i]:
2833                                     i = i - 1
2834                                     break
2835
2836                             paths.append ("/".join(cur_list[:i + 1]))
2837
2838                         self.previousDepotPaths = paths
2839
2840             if p4Change > 0:
2841                 self.depotPaths = sorted(self.previousDepotPaths)
2842                 self.changeRange = "@%s,#head" % p4Change
2843                 if not self.silent and not self.detectBranches:
2844                     print "Performing incremental import into %s git branch" % self.branch
2845
2846         # accept multiple ref name abbreviations:
2847         #    refs/foo/bar/branch -> use it exactly
2848         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2849         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2850         if not self.branch.startswith("refs/"):
2851             if self.importIntoRemotes:
2852                 prepend = "refs/remotes/"
2853             else:
2854                 prepend = "refs/heads/"
2855             if not self.branch.startswith("p4/"):
2856                 prepend += "p4/"
2857             self.branch = prepend + self.branch
2858
2859         if len(args) == 0 and self.depotPaths:
2860             if not self.silent:
2861                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2862         else:
2863             if self.depotPaths and self.depotPaths != args:
2864                 print ("previous import used depot path %s and now %s was specified. "
2865                        "This doesn't work!" % (' '.join (self.depotPaths),
2866                                                ' '.join (args)))
2867                 sys.exit(1)
2868
2869             self.depotPaths = sorted(args)
2870
2871         revision = ""
2872         self.users = {}
2873
2874         # Make sure no revision specifiers are used when --changesfile
2875         # is specified.
2876         bad_changesfile = False
2877         if len(self.changesFile) > 0:
2878             for p in self.depotPaths:
2879                 if p.find("@") >= 0 or p.find("#") >= 0:
2880                     bad_changesfile = True
2881                     break
2882         if bad_changesfile:
2883             die("Option --changesfile is incompatible with revision specifiers")
2884
2885         newPaths = []
2886         for p in self.depotPaths:
2887             if p.find("@") != -1:
2888                 atIdx = p.index("@")
2889                 self.changeRange = p[atIdx:]
2890                 if self.changeRange == "@all":
2891                     self.changeRange = ""
2892                 elif ',' not in self.changeRange:
2893                     revision = self.changeRange
2894                     self.changeRange = ""
2895                 p = p[:atIdx]
2896             elif p.find("#") != -1:
2897                 hashIdx = p.index("#")
2898                 revision = p[hashIdx:]
2899                 p = p[:hashIdx]
2900             elif self.previousDepotPaths == []:
2901                 # pay attention to changesfile, if given, else import
2902                 # the entire p4 tree at the head revision
2903                 if len(self.changesFile) == 0:
2904                     revision = "#head"
2905
2906             p = re.sub ("\.\.\.$", "", p)
2907             if not p.endswith("/"):
2908                 p += "/"
2909
2910             newPaths.append(p)
2911
2912         self.depotPaths = newPaths
2913
2914         # --detect-branches may change this for each branch
2915         self.branchPrefixes = self.depotPaths
2916
2917         self.loadUserMapFromCache()
2918         self.labels = {}
2919         if self.detectLabels:
2920             self.getLabels();
2921
2922         if self.detectBranches:
2923             ## FIXME - what's a P4 projectName ?
2924             self.projectName = self.guessProjectName()
2925
2926             if self.hasOrigin:
2927                 self.getBranchMappingFromGitBranches()
2928             else:
2929                 self.getBranchMapping()
2930             if self.verbose:
2931                 print "p4-git branches: %s" % self.p4BranchesInGit
2932                 print "initial parents: %s" % self.initialParents
2933             for b in self.p4BranchesInGit:
2934                 if b != "master":
2935
2936                     ## FIXME
2937                     b = b[len(self.projectName):]
2938                 self.createdBranches.add(b)
2939
2940         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2941
2942         self.importProcess = subprocess.Popen(["git", "fast-import"],
2943                                               stdin=subprocess.PIPE,
2944                                               stdout=subprocess.PIPE,
2945                                               stderr=subprocess.PIPE);
2946         self.gitOutput = self.importProcess.stdout
2947         self.gitStream = self.importProcess.stdin
2948         self.gitError = self.importProcess.stderr
2949
2950         if revision:
2951             self.importHeadRevision(revision)
2952         else:
2953             changes = []
2954
2955             if len(self.changesFile) > 0:
2956                 output = open(self.changesFile).readlines()
2957                 changeSet = set()
2958                 for line in output:
2959                     changeSet.add(int(line))
2960
2961                 for change in changeSet:
2962                     changes.append(change)
2963
2964                 changes.sort()
2965             else:
2966                 # catch "git p4 sync" with no new branches, in a repo that
2967                 # does not have any existing p4 branches
2968                 if len(args) == 0:
2969                     if not self.p4BranchesInGit:
2970                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
2971
2972                     # The default branch is master, unless --branch is used to
2973                     # specify something else.  Make sure it exists, or complain
2974                     # nicely about how to use --branch.
2975                     if not self.detectBranches:
2976                         if not branch_exists(self.branch):
2977                             if branch_arg_given:
2978                                 die("Error: branch %s does not exist." % self.branch)
2979                             else:
2980                                 die("Error: no branch %s; perhaps specify one with --branch." %
2981                                     self.branch)
2982
2983                 if self.verbose:
2984                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2985                                                               self.changeRange)
2986                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2987
2988                 if len(self.maxChanges) > 0:
2989                     changes = changes[:min(int(self.maxChanges), len(changes))]
2990
2991             if len(changes) == 0:
2992                 if not self.silent:
2993                     print "No changes to import!"
2994             else:
2995                 if not self.silent and not self.detectBranches:
2996                     print "Import destination: %s" % self.branch
2997
2998                 self.updatedBranches = set()
2999
3000                 if not self.detectBranches:
3001                     if args:
3002                         # start a new branch
3003                         self.initialParent = ""
3004                     else:
3005                         # build on a previous revision
3006                         self.initialParent = parseRevision(self.branch)
3007
3008                 self.importChanges(changes)
3009
3010                 if not self.silent:
3011                     print ""
3012                     if len(self.updatedBranches) > 0:
3013                         sys.stdout.write("Updated branches: ")
3014                         for b in self.updatedBranches:
3015                             sys.stdout.write("%s " % b)
3016                         sys.stdout.write("\n")
3017
3018         if gitConfigBool("git-p4.importLabels"):
3019             self.importLabels = True
3020
3021         if self.importLabels:
3022             p4Labels = getP4Labels(self.depotPaths)
3023             gitTags = getGitTags()
3024
3025             missingP4Labels = p4Labels - gitTags
3026             self.importP4Labels(self.gitStream, missingP4Labels)
3027
3028         self.gitStream.close()
3029         if self.importProcess.wait() != 0:
3030             die("fast-import failed: %s" % self.gitError.read())
3031         self.gitOutput.close()
3032         self.gitError.close()
3033
3034         # Cleanup temporary branches created during import
3035         if self.tempBranches != []:
3036             for branch in self.tempBranches:
3037                 read_pipe("git update-ref -d %s" % branch)
3038             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3039
3040         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3041         # a convenient shortcut refname "p4".
3042         if self.importIntoRemotes:
3043             head_ref = self.refPrefix + "HEAD"
3044             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3045                 system(["git", "symbolic-ref", head_ref, self.branch])
3046
3047         return True
3048
3049 class P4Rebase(Command):
3050     def __init__(self):
3051         Command.__init__(self)
3052         self.options = [
3053                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3054         ]
3055         self.importLabels = False
3056         self.description = ("Fetches the latest revision from perforce and "
3057                             + "rebases the current work (branch) against it")
3058
3059     def run(self, args):
3060         sync = P4Sync()
3061         sync.importLabels = self.importLabels
3062         sync.run([])
3063
3064         return self.rebase()
3065
3066     def rebase(self):
3067         if os.system("git update-index --refresh") != 0:
3068             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3069         if len(read_pipe("git diff-index HEAD --")) > 0:
3070             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3071
3072         [upstream, settings] = findUpstreamBranchPoint()
3073         if len(upstream) == 0:
3074             die("Cannot find upstream branchpoint for rebase")
3075
3076         # the branchpoint may be p4/foo~3, so strip off the parent
3077         upstream = re.sub("~[0-9]+$", "", upstream)
3078
3079         print "Rebasing the current branch onto %s" % upstream
3080         oldHead = read_pipe("git rev-parse HEAD").strip()
3081         system("git rebase %s" % upstream)
3082         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3083         return True
3084
3085 class P4Clone(P4Sync):
3086     def __init__(self):
3087         P4Sync.__init__(self)
3088         self.description = "Creates a new git repository and imports from Perforce into it"
3089         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3090         self.options += [
3091             optparse.make_option("--destination", dest="cloneDestination",
3092                                  action='store', default=None,
3093                                  help="where to leave result of the clone"),
3094             optparse.make_option("-/", dest="cloneExclude",
3095                                  action="append", type="string",
3096                                  help="exclude depot path"),
3097             optparse.make_option("--bare", dest="cloneBare",
3098                                  action="store_true", default=False),
3099         ]
3100         self.cloneDestination = None
3101         self.needsGit = False
3102         self.cloneBare = False
3103
3104     # This is required for the "append" cloneExclude action
3105     def ensure_value(self, attr, value):
3106         if not hasattr(self, attr) or getattr(self, attr) is None:
3107             setattr(self, attr, value)
3108         return getattr(self, attr)
3109
3110     def defaultDestination(self, args):
3111         ## TODO: use common prefix of args?
3112         depotPath = args[0]
3113         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3114         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3115         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3116         depotDir = re.sub(r"/$", "", depotDir)
3117         return os.path.split(depotDir)[1]
3118
3119     def run(self, args):
3120         if len(args) < 1:
3121             return False
3122
3123         if self.keepRepoPath and not self.cloneDestination:
3124             sys.stderr.write("Must specify destination for --keep-path\n")
3125             sys.exit(1)
3126
3127         depotPaths = args
3128
3129         if not self.cloneDestination and len(depotPaths) > 1:
3130             self.cloneDestination = depotPaths[-1]
3131             depotPaths = depotPaths[:-1]
3132
3133         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3134         for p in depotPaths:
3135             if not p.startswith("//"):
3136                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3137                 return False
3138
3139         if not self.cloneDestination:
3140             self.cloneDestination = self.defaultDestination(args)
3141
3142         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3143
3144         if not os.path.exists(self.cloneDestination):
3145             os.makedirs(self.cloneDestination)
3146         chdir(self.cloneDestination)
3147
3148         init_cmd = [ "git", "init" ]
3149         if self.cloneBare:
3150             init_cmd.append("--bare")
3151         retcode = subprocess.call(init_cmd)
3152         if retcode:
3153             raise CalledProcessError(retcode, init_cmd)
3154
3155         if not P4Sync.run(self, depotPaths):
3156             return False
3157
3158         # create a master branch and check out a work tree
3159         if gitBranchExists(self.branch):
3160             system([ "git", "branch", "master", self.branch ])
3161             if not self.cloneBare:
3162                 system([ "git", "checkout", "-f" ])
3163         else:
3164             print 'Not checking out any branch, use ' \
3165                   '"git checkout -q -b master <branch>"'
3166
3167         # auto-set this variable if invoked with --use-client-spec
3168         if self.useClientSpec_from_options:
3169             system("git config --bool git-p4.useclientspec true")
3170
3171         return True
3172
3173 class P4Branches(Command):
3174     def __init__(self):
3175         Command.__init__(self)
3176         self.options = [ ]
3177         self.description = ("Shows the git branches that hold imports and their "
3178                             + "corresponding perforce depot paths")
3179         self.verbose = False
3180
3181     def run(self, args):
3182         if originP4BranchesExist():
3183             createOrUpdateBranchesFromOrigin()
3184
3185         cmdline = "git rev-parse --symbolic "
3186         cmdline += " --remotes"
3187
3188         for line in read_pipe_lines(cmdline):
3189             line = line.strip()
3190
3191             if not line.startswith('p4/') or line == "p4/HEAD":
3192                 continue
3193             branch = line
3194
3195             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3196             settings = extractSettingsGitLog(log)
3197
3198             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3199         return True
3200
3201 class HelpFormatter(optparse.IndentedHelpFormatter):
3202     def __init__(self):
3203         optparse.IndentedHelpFormatter.__init__(self)
3204
3205     def format_description(self, description):
3206         if description:
3207             return description + "\n"
3208         else:
3209             return ""
3210
3211 def printUsage(commands):
3212     print "usage: %s <command> [options]" % sys.argv[0]
3213     print ""
3214     print "valid commands: %s" % ", ".join(commands)
3215     print ""
3216     print "Try %s <command> --help for command specific help." % sys.argv[0]
3217     print ""
3218
3219 commands = {
3220     "debug" : P4Debug,
3221     "submit" : P4Submit,
3222     "commit" : P4Submit,
3223     "sync" : P4Sync,
3224     "rebase" : P4Rebase,
3225     "clone" : P4Clone,
3226     "rollback" : P4RollBack,
3227     "branches" : P4Branches
3228 }
3229
3230
3231 def main():
3232     if len(sys.argv[1:]) == 0:
3233         printUsage(commands.keys())
3234         sys.exit(2)
3235
3236     cmdName = sys.argv[1]
3237     try:
3238         klass = commands[cmdName]
3239         cmd = klass()
3240     except KeyError:
3241         print "unknown command %s" % cmdName
3242         print ""
3243         printUsage(commands.keys())
3244         sys.exit(2)
3245
3246     options = cmd.options
3247     cmd.gitdir = os.environ.get("GIT_DIR", None)
3248
3249     args = sys.argv[2:]
3250
3251     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3252     if cmd.needsGit:
3253         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3254
3255     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3256                                    options,
3257                                    description = cmd.description,
3258                                    formatter = HelpFormatter())
3259
3260     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3261     global verbose
3262     verbose = cmd.verbose
3263     if cmd.needsGit:
3264         if cmd.gitdir == None:
3265             cmd.gitdir = os.path.abspath(".git")
3266             if not isValidGitDir(cmd.gitdir):
3267                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3268                 if os.path.exists(cmd.gitdir):
3269                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3270                     if len(cdup) > 0:
3271                         chdir(cdup);
3272
3273         if not isValidGitDir(cmd.gitdir):
3274             if isValidGitDir(cmd.gitdir + "/.git"):
3275                 cmd.gitdir += "/.git"
3276             else:
3277                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3278
3279         os.environ["GIT_DIR"] = cmd.gitdir
3280
3281     if not cmd.run(args):
3282         parser.print_help()
3283         sys.exit(2)
3284
3285
3286 if __name__ == '__main__':
3287     main()