Merge branch 'jk/pack-corruption-post-mortem'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27     from subprocess import CalledProcessError
28 except ImportError:
29     # from python2.7:subprocess.py
30     # Exception classes used by this module.
31     class CalledProcessError(Exception):
32         """This exception is raised when a process run by check_call() returns
33         a non-zero exit status.  The exit status will be stored in the
34         returncode attribute."""
35         def __init__(self, returncode, cmd):
36             self.returncode = returncode
37             self.cmd = cmd
38         def __str__(self):
39             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 def p4_build_cmd(cmd):
47     """Build a suitable p4 command line.
48
49     This consolidates building and returning a p4 command line into one
50     location. It means that hooking into the environment, or other configuration
51     can be done more easily.
52     """
53     real_cmd = ["p4"]
54
55     user = gitConfig("git-p4.user")
56     if len(user) > 0:
57         real_cmd += ["-u",user]
58
59     password = gitConfig("git-p4.password")
60     if len(password) > 0:
61         real_cmd += ["-P", password]
62
63     port = gitConfig("git-p4.port")
64     if len(port) > 0:
65         real_cmd += ["-p", port]
66
67     host = gitConfig("git-p4.host")
68     if len(host) > 0:
69         real_cmd += ["-H", host]
70
71     client = gitConfig("git-p4.client")
72     if len(client) > 0:
73         real_cmd += ["-c", client]
74
75
76     if isinstance(cmd,basestring):
77         real_cmd = ' '.join(real_cmd) + ' ' + cmd
78     else:
79         real_cmd += cmd
80     return real_cmd
81
82 def chdir(path, is_client_path=False):
83     """Do chdir to the given path, and set the PWD environment
84        variable for use by P4.  It does not look at getcwd() output.
85        Since we're not using the shell, it is necessary to set the
86        PWD environment variable explicitly.
87
88        Normally, expand the path to force it to be absolute.  This
89        addresses the use of relative path names inside P4 settings,
90        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
91        as given; it looks for .p4config using PWD.
92
93        If is_client_path, the path was handed to us directly by p4,
94        and may be a symbolic link.  Do not call os.getcwd() in this
95        case, because it will cause p4 to think that PWD is not inside
96        the client path.
97        """
98
99     os.chdir(path)
100     if not is_client_path:
101         path = os.getcwd()
102     os.environ['PWD'] = path
103
104 def die(msg):
105     if verbose:
106         raise Exception(msg)
107     else:
108         sys.stderr.write(msg + "\n")
109         sys.exit(1)
110
111 def write_pipe(c, stdin):
112     if verbose:
113         sys.stderr.write('Writing pipe: %s\n' % str(c))
114
115     expand = isinstance(c,basestring)
116     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117     pipe = p.stdin
118     val = pipe.write(stdin)
119     pipe.close()
120     if p.wait():
121         die('Command failed: %s' % str(c))
122
123     return val
124
125 def p4_write_pipe(c, stdin):
126     real_cmd = p4_build_cmd(c)
127     return write_pipe(real_cmd, stdin)
128
129 def read_pipe(c, ignore_error=False):
130     if verbose:
131         sys.stderr.write('Reading pipe: %s\n' % str(c))
132
133     expand = isinstance(c,basestring)
134     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135     pipe = p.stdout
136     val = pipe.read()
137     if p.wait() and not ignore_error:
138         die('Command failed: %s' % str(c))
139
140     return val
141
142 def p4_read_pipe(c, ignore_error=False):
143     real_cmd = p4_build_cmd(c)
144     return read_pipe(real_cmd, ignore_error)
145
146 def read_pipe_lines(c):
147     if verbose:
148         sys.stderr.write('Reading pipe: %s\n' % str(c))
149
150     expand = isinstance(c, basestring)
151     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152     pipe = p.stdout
153     val = pipe.readlines()
154     if pipe.close() or p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_read_pipe_lines(c):
160     """Specifically invoke p4 on the command supplied. """
161     real_cmd = p4_build_cmd(c)
162     return read_pipe_lines(real_cmd)
163
164 def p4_has_command(cmd):
165     """Ask p4 for help on this command.  If it returns an error, the
166        command does not exist in this version of p4."""
167     real_cmd = p4_build_cmd(["help", cmd])
168     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169                                    stderr=subprocess.PIPE)
170     p.communicate()
171     return p.returncode == 0
172
173 def p4_has_move_command():
174     """See if the move command exists, that it supports -k, and that
175        it has not been administratively disabled.  The arguments
176        must be correct, but the filenames do not have to exist.  Use
177        ones with wildcards so even if they exist, it will fail."""
178
179     if not p4_has_command("move"):
180         return False
181     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183     (out, err) = p.communicate()
184     # return code will be 1 in either case
185     if err.find("Invalid option") >= 0:
186         return False
187     if err.find("disabled") >= 0:
188         return False
189     # assume it failed because @... was invalid changelist
190     return True
191
192 def system(cmd):
193     expand = isinstance(cmd,basestring)
194     if verbose:
195         sys.stderr.write("executing %s\n" % str(cmd))
196     retcode = subprocess.call(cmd, shell=expand)
197     if retcode:
198         raise CalledProcessError(retcode, cmd)
199
200 def p4_system(cmd):
201     """Specifically invoke p4 as the system command. """
202     real_cmd = p4_build_cmd(cmd)
203     expand = isinstance(real_cmd, basestring)
204     retcode = subprocess.call(real_cmd, shell=expand)
205     if retcode:
206         raise CalledProcessError(retcode, real_cmd)
207
208 _p4_version_string = None
209 def p4_version_string():
210     """Read the version string, showing just the last line, which
211        hopefully is the interesting version bit.
212
213        $ p4 -V
214        Perforce - The Fast Software Configuration Management System.
215        Copyright 1995-2011 Perforce Software.  All rights reserved.
216        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
217     """
218     global _p4_version_string
219     if not _p4_version_string:
220         a = p4_read_pipe_lines(["-V"])
221         _p4_version_string = a[-1].rstrip()
222     return _p4_version_string
223
224 def p4_integrate(src, dest):
225     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
226
227 def p4_sync(f, *options):
228     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
229
230 def p4_add(f):
231     # forcibly add file names with wildcards
232     if wildcard_present(f):
233         p4_system(["add", "-f", f])
234     else:
235         p4_system(["add", f])
236
237 def p4_delete(f):
238     p4_system(["delete", wildcard_encode(f)])
239
240 def p4_edit(f):
241     p4_system(["edit", wildcard_encode(f)])
242
243 def p4_revert(f):
244     p4_system(["revert", wildcard_encode(f)])
245
246 def p4_reopen(type, f):
247     p4_system(["reopen", "-t", type, wildcard_encode(f)])
248
249 def p4_move(src, dest):
250     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
251
252 def p4_describe(change):
253     """Make sure it returns a valid result by checking for
254        the presence of field "time".  Return a dict of the
255        results."""
256
257     ds = p4CmdList(["describe", "-s", str(change)])
258     if len(ds) != 1:
259         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
260
261     d = ds[0]
262
263     if "p4ExitCode" in d:
264         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265                                                       str(d)))
266     if "code" in d:
267         if d["code"] == "error":
268             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
269
270     if "time" not in d:
271         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
272
273     return d
274
275 #
276 # Canonicalize the p4 type and return a tuple of the
277 # base type, plus any modifiers.  See "p4 help filetypes"
278 # for a list and explanation.
279 #
280 def split_p4_type(p4type):
281
282     p4_filetypes_historical = {
283         "ctempobj": "binary+Sw",
284         "ctext": "text+C",
285         "cxtext": "text+Cx",
286         "ktext": "text+k",
287         "kxtext": "text+kx",
288         "ltext": "text+F",
289         "tempobj": "binary+FSw",
290         "ubinary": "binary+F",
291         "uresource": "resource+F",
292         "uxbinary": "binary+Fx",
293         "xbinary": "binary+x",
294         "xltext": "text+Fx",
295         "xtempobj": "binary+Swx",
296         "xtext": "text+x",
297         "xunicode": "unicode+x",
298         "xutf16": "utf16+x",
299     }
300     if p4type in p4_filetypes_historical:
301         p4type = p4_filetypes_historical[p4type]
302     mods = ""
303     s = p4type.split("+")
304     base = s[0]
305     mods = ""
306     if len(s) > 1:
307         mods = s[1]
308     return (base, mods)
309
310 #
311 # return the raw p4 type of a file (text, text+ko, etc)
312 #
313 def p4_type(f):
314     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
315     return results[0]['headType']
316
317 #
318 # Given a type base and modifier, return a regexp matching
319 # the keywords that can be expanded in the file
320 #
321 def p4_keywords_regexp_for_type(base, type_mods):
322     if base in ("text", "unicode", "binary"):
323         kwords = None
324         if "ko" in type_mods:
325             kwords = 'Id|Header'
326         elif "k" in type_mods:
327             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328         else:
329             return None
330         pattern = r"""
331             \$              # Starts with a dollar, followed by...
332             (%s)            # one of the keywords, followed by...
333             (:[^$\n]+)?     # possibly an old expansion, followed by...
334             \$              # another dollar
335             """ % kwords
336         return pattern
337     else:
338         return None
339
340 #
341 # Given a file, return a regexp matching the possible
342 # RCS keywords that will be expanded, or None for files
343 # with kw expansion turned off.
344 #
345 def p4_keywords_regexp_for_file(file):
346     if not os.path.exists(file):
347         return None
348     else:
349         (type_base, type_mods) = split_p4_type(p4_type(file))
350         return p4_keywords_regexp_for_type(type_base, type_mods)
351
352 def setP4ExecBit(file, mode):
353     # Reopens an already open file and changes the execute bit to match
354     # the execute bit setting in the passed in mode.
355
356     p4Type = "+x"
357
358     if not isModeExec(mode):
359         p4Type = getP4OpenedType(file)
360         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362         if p4Type[-1] == "+":
363             p4Type = p4Type[0:-1]
364
365     p4_reopen(p4Type, file)
366
367 def getP4OpenedType(file):
368     # Returns the perforce file type for the given file.
369
370     result = p4_read_pipe(["opened", wildcard_encode(file)])
371     match = re.match(".*\((.+)\)\r?$", result)
372     if match:
373         return match.group(1)
374     else:
375         die("Could not determine file type for %s (result: '%s')" % (file, result))
376
377 # Return the set of all p4 labels
378 def getP4Labels(depotPaths):
379     labels = set()
380     if isinstance(depotPaths,basestring):
381         depotPaths = [depotPaths]
382
383     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384         label = l['label']
385         labels.add(label)
386
387     return labels
388
389 # Return the set of all git tags
390 def getGitTags():
391     gitTags = set()
392     for line in read_pipe_lines(["git", "tag"]):
393         tag = line.strip()
394         gitTags.add(tag)
395     return gitTags
396
397 def diffTreePattern():
398     # This is a simple generator for the diff tree regex pattern. This could be
399     # a class variable if this and parseDiffTreeEntry were a part of a class.
400     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401     while True:
402         yield pattern
403
404 def parseDiffTreeEntry(entry):
405     """Parses a single diff tree entry into its component elements.
406
407     See git-diff-tree(1) manpage for details about the format of the diff
408     output. This method returns a dictionary with the following elements:
409
410     src_mode - The mode of the source file
411     dst_mode - The mode of the destination file
412     src_sha1 - The sha1 for the source file
413     dst_sha1 - The sha1 fr the destination file
414     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415     status_score - The score for the status (applicable for 'C' and 'R'
416                    statuses). This is None if there is no score.
417     src - The path for the source file.
418     dst - The path for the destination file. This is only present for
419           copy or renames. If it is not present, this is None.
420
421     If the pattern is not matched, None is returned."""
422
423     match = diffTreePattern().next().match(entry)
424     if match:
425         return {
426             'src_mode': match.group(1),
427             'dst_mode': match.group(2),
428             'src_sha1': match.group(3),
429             'dst_sha1': match.group(4),
430             'status': match.group(5),
431             'status_score': match.group(6),
432             'src': match.group(7),
433             'dst': match.group(10)
434         }
435     return None
436
437 def isModeExec(mode):
438     # Returns True if the given git mode represents an executable file,
439     # otherwise False.
440     return mode[-3:] == "755"
441
442 def isModeExecChanged(src_mode, dst_mode):
443     return isModeExec(src_mode) != isModeExec(dst_mode)
444
445 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
446
447     if isinstance(cmd,basestring):
448         cmd = "-G " + cmd
449         expand = True
450     else:
451         cmd = ["-G"] + cmd
452         expand = False
453
454     cmd = p4_build_cmd(cmd)
455     if verbose:
456         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
457
458     # Use a temporary file to avoid deadlocks without
459     # subprocess.communicate(), which would put another copy
460     # of stdout into memory.
461     stdin_file = None
462     if stdin is not None:
463         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
464         if isinstance(stdin,basestring):
465             stdin_file.write(stdin)
466         else:
467             for i in stdin:
468                 stdin_file.write(i + '\n')
469         stdin_file.flush()
470         stdin_file.seek(0)
471
472     p4 = subprocess.Popen(cmd,
473                           shell=expand,
474                           stdin=stdin_file,
475                           stdout=subprocess.PIPE)
476
477     result = []
478     try:
479         while True:
480             entry = marshal.load(p4.stdout)
481             if cb is not None:
482                 cb(entry)
483             else:
484                 result.append(entry)
485     except EOFError:
486         pass
487     exitCode = p4.wait()
488     if exitCode != 0:
489         entry = {}
490         entry["p4ExitCode"] = exitCode
491         result.append(entry)
492
493     return result
494
495 def p4Cmd(cmd):
496     list = p4CmdList(cmd)
497     result = {}
498     for entry in list:
499         result.update(entry)
500     return result;
501
502 def p4Where(depotPath):
503     if not depotPath.endswith("/"):
504         depotPath += "/"
505     depotPath = depotPath + "..."
506     outputList = p4CmdList(["where", depotPath])
507     output = None
508     for entry in outputList:
509         if "depotFile" in entry:
510             if entry["depotFile"] == depotPath:
511                 output = entry
512                 break
513         elif "data" in entry:
514             data = entry.get("data")
515             space = data.find(" ")
516             if data[:space] == depotPath:
517                 output = entry
518                 break
519     if output == None:
520         return ""
521     if output["code"] == "error":
522         return ""
523     clientPath = ""
524     if "path" in output:
525         clientPath = output.get("path")
526     elif "data" in output:
527         data = output.get("data")
528         lastSpace = data.rfind(" ")
529         clientPath = data[lastSpace + 1:]
530
531     if clientPath.endswith("..."):
532         clientPath = clientPath[:-3]
533     return clientPath
534
535 def currentGitBranch():
536     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
537
538 def isValidGitDir(path):
539     if (os.path.exists(path + "/HEAD")
540         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
541         return True;
542     return False
543
544 def parseRevision(ref):
545     return read_pipe("git rev-parse %s" % ref).strip()
546
547 def branchExists(ref):
548     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
549                      ignore_error=True)
550     return len(rev) > 0
551
552 def extractLogMessageFromGitCommit(commit):
553     logMessage = ""
554
555     ## fixme: title is first line of commit, not 1st paragraph.
556     foundTitle = False
557     for log in read_pipe_lines("git cat-file commit %s" % commit):
558        if not foundTitle:
559            if len(log) == 1:
560                foundTitle = True
561            continue
562
563        logMessage += log
564     return logMessage
565
566 def extractSettingsGitLog(log):
567     values = {}
568     for line in log.split("\n"):
569         line = line.strip()
570         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
571         if not m:
572             continue
573
574         assignments = m.group(1).split (':')
575         for a in assignments:
576             vals = a.split ('=')
577             key = vals[0].strip()
578             val = ('='.join (vals[1:])).strip()
579             if val.endswith ('\"') and val.startswith('"'):
580                 val = val[1:-1]
581
582             values[key] = val
583
584     paths = values.get("depot-paths")
585     if not paths:
586         paths = values.get("depot-path")
587     if paths:
588         values['depot-paths'] = paths.split(',')
589     return values
590
591 def gitBranchExists(branch):
592     proc = subprocess.Popen(["git", "rev-parse", branch],
593                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
594     return proc.wait() == 0;
595
596 _gitConfig = {}
597
598 def gitConfig(key):
599     if not _gitConfig.has_key(key):
600         cmd = [ "git", "config", key ]
601         s = read_pipe(cmd, ignore_error=True)
602         _gitConfig[key] = s.strip()
603     return _gitConfig[key]
604
605 def gitConfigBool(key):
606     """Return a bool, using git config --bool.  It is True only if the
607        variable is set to true, and False if set to false or not present
608        in the config."""
609
610     if not _gitConfig.has_key(key):
611         cmd = [ "git", "config", "--bool", key ]
612         s = read_pipe(cmd, ignore_error=True)
613         v = s.strip()
614         _gitConfig[key] = v == "true"
615     return _gitConfig[key]
616
617 def gitConfigList(key):
618     if not _gitConfig.has_key(key):
619         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
620         _gitConfig[key] = s.strip().split(os.linesep)
621     return _gitConfig[key]
622
623 def p4BranchesInGit(branchesAreInRemotes=True):
624     """Find all the branches whose names start with "p4/", looking
625        in remotes or heads as specified by the argument.  Return
626        a dictionary of { branch: revision } for each one found.
627        The branch names are the short names, without any
628        "p4/" prefix."""
629
630     branches = {}
631
632     cmdline = "git rev-parse --symbolic "
633     if branchesAreInRemotes:
634         cmdline += "--remotes"
635     else:
636         cmdline += "--branches"
637
638     for line in read_pipe_lines(cmdline):
639         line = line.strip()
640
641         # only import to p4/
642         if not line.startswith('p4/'):
643             continue
644         # special symbolic ref to p4/master
645         if line == "p4/HEAD":
646             continue
647
648         # strip off p4/ prefix
649         branch = line[len("p4/"):]
650
651         branches[branch] = parseRevision(line)
652
653     return branches
654
655 def branch_exists(branch):
656     """Make sure that the given ref name really exists."""
657
658     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
659     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
660     out, _ = p.communicate()
661     if p.returncode:
662         return False
663     # expect exactly one line of output: the branch name
664     return out.rstrip() == branch
665
666 def findUpstreamBranchPoint(head = "HEAD"):
667     branches = p4BranchesInGit()
668     # map from depot-path to branch name
669     branchByDepotPath = {}
670     for branch in branches.keys():
671         tip = branches[branch]
672         log = extractLogMessageFromGitCommit(tip)
673         settings = extractSettingsGitLog(log)
674         if settings.has_key("depot-paths"):
675             paths = ",".join(settings["depot-paths"])
676             branchByDepotPath[paths] = "remotes/p4/" + branch
677
678     settings = None
679     parent = 0
680     while parent < 65535:
681         commit = head + "~%s" % parent
682         log = extractLogMessageFromGitCommit(commit)
683         settings = extractSettingsGitLog(log)
684         if settings.has_key("depot-paths"):
685             paths = ",".join(settings["depot-paths"])
686             if branchByDepotPath.has_key(paths):
687                 return [branchByDepotPath[paths], settings]
688
689         parent = parent + 1
690
691     return ["", settings]
692
693 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
694     if not silent:
695         print ("Creating/updating branch(es) in %s based on origin branch(es)"
696                % localRefPrefix)
697
698     originPrefix = "origin/p4/"
699
700     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
701         line = line.strip()
702         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
703             continue
704
705         headName = line[len(originPrefix):]
706         remoteHead = localRefPrefix + headName
707         originHead = line
708
709         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
710         if (not original.has_key('depot-paths')
711             or not original.has_key('change')):
712             continue
713
714         update = False
715         if not gitBranchExists(remoteHead):
716             if verbose:
717                 print "creating %s" % remoteHead
718             update = True
719         else:
720             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
721             if settings.has_key('change') > 0:
722                 if settings['depot-paths'] == original['depot-paths']:
723                     originP4Change = int(original['change'])
724                     p4Change = int(settings['change'])
725                     if originP4Change > p4Change:
726                         print ("%s (%s) is newer than %s (%s). "
727                                "Updating p4 branch from origin."
728                                % (originHead, originP4Change,
729                                   remoteHead, p4Change))
730                         update = True
731                 else:
732                     print ("Ignoring: %s was imported from %s while "
733                            "%s was imported from %s"
734                            % (originHead, ','.join(original['depot-paths']),
735                               remoteHead, ','.join(settings['depot-paths'])))
736
737         if update:
738             system("git update-ref %s %s" % (remoteHead, originHead))
739
740 def originP4BranchesExist():
741         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
742
743 def p4ChangesForPaths(depotPaths, changeRange):
744     assert depotPaths
745     cmd = ['changes']
746     for p in depotPaths:
747         cmd += ["%s...%s" % (p, changeRange)]
748     output = p4_read_pipe_lines(cmd)
749
750     changes = {}
751     for line in output:
752         changeNum = int(line.split(" ")[1])
753         changes[changeNum] = True
754
755     changelist = changes.keys()
756     changelist.sort()
757     return changelist
758
759 def p4PathStartsWith(path, prefix):
760     # This method tries to remedy a potential mixed-case issue:
761     #
762     # If UserA adds  //depot/DirA/file1
763     # and UserB adds //depot/dira/file2
764     #
765     # we may or may not have a problem. If you have core.ignorecase=true,
766     # we treat DirA and dira as the same directory
767     if gitConfigBool("core.ignorecase"):
768         return path.lower().startswith(prefix.lower())
769     return path.startswith(prefix)
770
771 def getClientSpec():
772     """Look at the p4 client spec, create a View() object that contains
773        all the mappings, and return it."""
774
775     specList = p4CmdList("client -o")
776     if len(specList) != 1:
777         die('Output from "client -o" is %d lines, expecting 1' %
778             len(specList))
779
780     # dictionary of all client parameters
781     entry = specList[0]
782
783     # the //client/ name
784     client_name = entry["Client"]
785
786     # just the keys that start with "View"
787     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
788
789     # hold this new View
790     view = View(client_name)
791
792     # append the lines, in order, to the view
793     for view_num in range(len(view_keys)):
794         k = "View%d" % view_num
795         if k not in view_keys:
796             die("Expected view key %s missing" % k)
797         view.append(entry[k])
798
799     return view
800
801 def getClientRoot():
802     """Grab the client directory."""
803
804     output = p4CmdList("client -o")
805     if len(output) != 1:
806         die('Output from "client -o" is %d lines, expecting 1' % len(output))
807
808     entry = output[0]
809     if "Root" not in entry:
810         die('Client has no "Root"')
811
812     return entry["Root"]
813
814 #
815 # P4 wildcards are not allowed in filenames.  P4 complains
816 # if you simply add them, but you can force it with "-f", in
817 # which case it translates them into %xx encoding internally.
818 #
819 def wildcard_decode(path):
820     # Search for and fix just these four characters.  Do % last so
821     # that fixing it does not inadvertently create new %-escapes.
822     # Cannot have * in a filename in windows; untested as to
823     # what p4 would do in such a case.
824     if not platform.system() == "Windows":
825         path = path.replace("%2A", "*")
826     path = path.replace("%23", "#") \
827                .replace("%40", "@") \
828                .replace("%25", "%")
829     return path
830
831 def wildcard_encode(path):
832     # do % first to avoid double-encoding the %s introduced here
833     path = path.replace("%", "%25") \
834                .replace("*", "%2A") \
835                .replace("#", "%23") \
836                .replace("@", "%40")
837     return path
838
839 def wildcard_present(path):
840     m = re.search("[*#@%]", path)
841     return m is not None
842
843 class Command:
844     def __init__(self):
845         self.usage = "usage: %prog [options]"
846         self.needsGit = True
847         self.verbose = False
848
849 class P4UserMap:
850     def __init__(self):
851         self.userMapFromPerforceServer = False
852         self.myP4UserId = None
853
854     def p4UserId(self):
855         if self.myP4UserId:
856             return self.myP4UserId
857
858         results = p4CmdList("user -o")
859         for r in results:
860             if r.has_key('User'):
861                 self.myP4UserId = r['User']
862                 return r['User']
863         die("Could not find your p4 user id")
864
865     def p4UserIsMe(self, p4User):
866         # return True if the given p4 user is actually me
867         me = self.p4UserId()
868         if not p4User or p4User != me:
869             return False
870         else:
871             return True
872
873     def getUserCacheFilename(self):
874         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
875         return home + "/.gitp4-usercache.txt"
876
877     def getUserMapFromPerforceServer(self):
878         if self.userMapFromPerforceServer:
879             return
880         self.users = {}
881         self.emails = {}
882
883         for output in p4CmdList("users"):
884             if not output.has_key("User"):
885                 continue
886             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
887             self.emails[output["Email"]] = output["User"]
888
889
890         s = ''
891         for (key, val) in self.users.items():
892             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
893
894         open(self.getUserCacheFilename(), "wb").write(s)
895         self.userMapFromPerforceServer = True
896
897     def loadUserMapFromCache(self):
898         self.users = {}
899         self.userMapFromPerforceServer = False
900         try:
901             cache = open(self.getUserCacheFilename(), "rb")
902             lines = cache.readlines()
903             cache.close()
904             for line in lines:
905                 entry = line.strip().split("\t")
906                 self.users[entry[0]] = entry[1]
907         except IOError:
908             self.getUserMapFromPerforceServer()
909
910 class P4Debug(Command):
911     def __init__(self):
912         Command.__init__(self)
913         self.options = []
914         self.description = "A tool to debug the output of p4 -G."
915         self.needsGit = False
916
917     def run(self, args):
918         j = 0
919         for output in p4CmdList(args):
920             print 'Element: %d' % j
921             j += 1
922             print output
923         return True
924
925 class P4RollBack(Command):
926     def __init__(self):
927         Command.__init__(self)
928         self.options = [
929             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
930         ]
931         self.description = "A tool to debug the multi-branch import. Don't use :)"
932         self.rollbackLocalBranches = False
933
934     def run(self, args):
935         if len(args) != 1:
936             return False
937         maxChange = int(args[0])
938
939         if "p4ExitCode" in p4Cmd("changes -m 1"):
940             die("Problems executing p4");
941
942         if self.rollbackLocalBranches:
943             refPrefix = "refs/heads/"
944             lines = read_pipe_lines("git rev-parse --symbolic --branches")
945         else:
946             refPrefix = "refs/remotes/"
947             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
948
949         for line in lines:
950             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
951                 line = line.strip()
952                 ref = refPrefix + line
953                 log = extractLogMessageFromGitCommit(ref)
954                 settings = extractSettingsGitLog(log)
955
956                 depotPaths = settings['depot-paths']
957                 change = settings['change']
958
959                 changed = False
960
961                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
962                                                            for p in depotPaths]))) == 0:
963                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
964                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
965                     continue
966
967                 while change and int(change) > maxChange:
968                     changed = True
969                     if self.verbose:
970                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
971                     system("git update-ref %s \"%s^\"" % (ref, ref))
972                     log = extractLogMessageFromGitCommit(ref)
973                     settings =  extractSettingsGitLog(log)
974
975
976                     depotPaths = settings['depot-paths']
977                     change = settings['change']
978
979                 if changed:
980                     print "%s rewound to %s" % (ref, change)
981
982         return True
983
984 class P4Submit(Command, P4UserMap):
985
986     conflict_behavior_choices = ("ask", "skip", "quit")
987
988     def __init__(self):
989         Command.__init__(self)
990         P4UserMap.__init__(self)
991         self.options = [
992                 optparse.make_option("--origin", dest="origin"),
993                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
994                 # preserve the user, requires relevant p4 permissions
995                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
996                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
997                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
998                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
999                 optparse.make_option("--conflict", dest="conflict_behavior",
1000                                      choices=self.conflict_behavior_choices),
1001                 optparse.make_option("--branch", dest="branch"),
1002         ]
1003         self.description = "Submit changes from git to the perforce depot."
1004         self.usage += " [name of git branch to submit into perforce depot]"
1005         self.origin = ""
1006         self.detectRenames = False
1007         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1008         self.dry_run = False
1009         self.prepare_p4_only = False
1010         self.conflict_behavior = None
1011         self.isWindows = (platform.system() == "Windows")
1012         self.exportLabels = False
1013         self.p4HasMoveCommand = p4_has_move_command()
1014         self.branch = None
1015
1016     def check(self):
1017         if len(p4CmdList("opened ...")) > 0:
1018             die("You have files opened with perforce! Close them before starting the sync.")
1019
1020     def separate_jobs_from_description(self, message):
1021         """Extract and return a possible Jobs field in the commit
1022            message.  It goes into a separate section in the p4 change
1023            specification.
1024
1025            A jobs line starts with "Jobs:" and looks like a new field
1026            in a form.  Values are white-space separated on the same
1027            line or on following lines that start with a tab.
1028
1029            This does not parse and extract the full git commit message
1030            like a p4 form.  It just sees the Jobs: line as a marker
1031            to pass everything from then on directly into the p4 form,
1032            but outside the description section.
1033
1034            Return a tuple (stripped log message, jobs string)."""
1035
1036         m = re.search(r'^Jobs:', message, re.MULTILINE)
1037         if m is None:
1038             return (message, None)
1039
1040         jobtext = message[m.start():]
1041         stripped_message = message[:m.start()].rstrip()
1042         return (stripped_message, jobtext)
1043
1044     def prepareLogMessage(self, template, message, jobs):
1045         """Edits the template returned from "p4 change -o" to insert
1046            the message in the Description field, and the jobs text in
1047            the Jobs field."""
1048         result = ""
1049
1050         inDescriptionSection = False
1051
1052         for line in template.split("\n"):
1053             if line.startswith("#"):
1054                 result += line + "\n"
1055                 continue
1056
1057             if inDescriptionSection:
1058                 if line.startswith("Files:") or line.startswith("Jobs:"):
1059                     inDescriptionSection = False
1060                     # insert Jobs section
1061                     if jobs:
1062                         result += jobs + "\n"
1063                 else:
1064                     continue
1065             else:
1066                 if line.startswith("Description:"):
1067                     inDescriptionSection = True
1068                     line += "\n"
1069                     for messageLine in message.split("\n"):
1070                         line += "\t" + messageLine + "\n"
1071
1072             result += line + "\n"
1073
1074         return result
1075
1076     def patchRCSKeywords(self, file, pattern):
1077         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1078         (handle, outFileName) = tempfile.mkstemp(dir='.')
1079         try:
1080             outFile = os.fdopen(handle, "w+")
1081             inFile = open(file, "r")
1082             regexp = re.compile(pattern, re.VERBOSE)
1083             for line in inFile.readlines():
1084                 line = regexp.sub(r'$\1$', line)
1085                 outFile.write(line)
1086             inFile.close()
1087             outFile.close()
1088             # Forcibly overwrite the original file
1089             os.unlink(file)
1090             shutil.move(outFileName, file)
1091         except:
1092             # cleanup our temporary file
1093             os.unlink(outFileName)
1094             print "Failed to strip RCS keywords in %s" % file
1095             raise
1096
1097         print "Patched up RCS keywords in %s" % file
1098
1099     def p4UserForCommit(self,id):
1100         # Return the tuple (perforce user,git email) for a given git commit id
1101         self.getUserMapFromPerforceServer()
1102         gitEmail = read_pipe(["git", "log", "--max-count=1",
1103                               "--format=%ae", id])
1104         gitEmail = gitEmail.strip()
1105         if not self.emails.has_key(gitEmail):
1106             return (None,gitEmail)
1107         else:
1108             return (self.emails[gitEmail],gitEmail)
1109
1110     def checkValidP4Users(self,commits):
1111         # check if any git authors cannot be mapped to p4 users
1112         for id in commits:
1113             (user,email) = self.p4UserForCommit(id)
1114             if not user:
1115                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1116                 if gitConfigBool("git-p4.allowMissingP4Users"):
1117                     print "%s" % msg
1118                 else:
1119                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1120
1121     def lastP4Changelist(self):
1122         # Get back the last changelist number submitted in this client spec. This
1123         # then gets used to patch up the username in the change. If the same
1124         # client spec is being used by multiple processes then this might go
1125         # wrong.
1126         results = p4CmdList("client -o")        # find the current client
1127         client = None
1128         for r in results:
1129             if r.has_key('Client'):
1130                 client = r['Client']
1131                 break
1132         if not client:
1133             die("could not get client spec")
1134         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1135         for r in results:
1136             if r.has_key('change'):
1137                 return r['change']
1138         die("Could not get changelist number for last submit - cannot patch up user details")
1139
1140     def modifyChangelistUser(self, changelist, newUser):
1141         # fixup the user field of a changelist after it has been submitted.
1142         changes = p4CmdList("change -o %s" % changelist)
1143         if len(changes) != 1:
1144             die("Bad output from p4 change modifying %s to user %s" %
1145                 (changelist, newUser))
1146
1147         c = changes[0]
1148         if c['User'] == newUser: return   # nothing to do
1149         c['User'] = newUser
1150         input = marshal.dumps(c)
1151
1152         result = p4CmdList("change -f -i", stdin=input)
1153         for r in result:
1154             if r.has_key('code'):
1155                 if r['code'] == 'error':
1156                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1157             if r.has_key('data'):
1158                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1159                 return
1160         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1161
1162     def canChangeChangelists(self):
1163         # check to see if we have p4 admin or super-user permissions, either of
1164         # which are required to modify changelists.
1165         results = p4CmdList(["protects", self.depotPath])
1166         for r in results:
1167             if r.has_key('perm'):
1168                 if r['perm'] == 'admin':
1169                     return 1
1170                 if r['perm'] == 'super':
1171                     return 1
1172         return 0
1173
1174     def prepareSubmitTemplate(self):
1175         """Run "p4 change -o" to grab a change specification template.
1176            This does not use "p4 -G", as it is nice to keep the submission
1177            template in original order, since a human might edit it.
1178
1179            Remove lines in the Files section that show changes to files
1180            outside the depot path we're committing into."""
1181
1182         template = ""
1183         inFilesSection = False
1184         for line in p4_read_pipe_lines(['change', '-o']):
1185             if line.endswith("\r\n"):
1186                 line = line[:-2] + "\n"
1187             if inFilesSection:
1188                 if line.startswith("\t"):
1189                     # path starts and ends with a tab
1190                     path = line[1:]
1191                     lastTab = path.rfind("\t")
1192                     if lastTab != -1:
1193                         path = path[:lastTab]
1194                         if not p4PathStartsWith(path, self.depotPath):
1195                             continue
1196                 else:
1197                     inFilesSection = False
1198             else:
1199                 if line.startswith("Files:"):
1200                     inFilesSection = True
1201
1202             template += line
1203
1204         return template
1205
1206     def edit_template(self, template_file):
1207         """Invoke the editor to let the user change the submission
1208            message.  Return true if okay to continue with the submit."""
1209
1210         # if configured to skip the editing part, just submit
1211         if gitConfigBool("git-p4.skipSubmitEdit"):
1212             return True
1213
1214         # look at the modification time, to check later if the user saved
1215         # the file
1216         mtime = os.stat(template_file).st_mtime
1217
1218         # invoke the editor
1219         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1220             editor = os.environ.get("P4EDITOR")
1221         else:
1222             editor = read_pipe("git var GIT_EDITOR").strip()
1223         system([editor, template_file])
1224
1225         # If the file was not saved, prompt to see if this patch should
1226         # be skipped.  But skip this verification step if configured so.
1227         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1228             return True
1229
1230         # modification time updated means user saved the file
1231         if os.stat(template_file).st_mtime > mtime:
1232             return True
1233
1234         while True:
1235             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1236             if response == 'y':
1237                 return True
1238             if response == 'n':
1239                 return False
1240
1241     def get_diff_description(self, editedFiles, filesToAdd):
1242         # diff
1243         if os.environ.has_key("P4DIFF"):
1244             del(os.environ["P4DIFF"])
1245         diff = ""
1246         for editedFile in editedFiles:
1247             diff += p4_read_pipe(['diff', '-du',
1248                                   wildcard_encode(editedFile)])
1249
1250         # new file diff
1251         newdiff = ""
1252         for newFile in filesToAdd:
1253             newdiff += "==== new file ====\n"
1254             newdiff += "--- /dev/null\n"
1255             newdiff += "+++ %s\n" % newFile
1256             f = open(newFile, "r")
1257             for line in f.readlines():
1258                 newdiff += "+" + line
1259             f.close()
1260
1261         return (diff + newdiff).replace('\r\n', '\n')
1262
1263     def applyCommit(self, id):
1264         """Apply one commit, return True if it succeeded."""
1265
1266         print "Applying", read_pipe(["git", "show", "-s",
1267                                      "--format=format:%h %s", id])
1268
1269         (p4User, gitEmail) = self.p4UserForCommit(id)
1270
1271         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1272         filesToAdd = set()
1273         filesToDelete = set()
1274         editedFiles = set()
1275         pureRenameCopy = set()
1276         filesToChangeExecBit = {}
1277
1278         for line in diff:
1279             diff = parseDiffTreeEntry(line)
1280             modifier = diff['status']
1281             path = diff['src']
1282             if modifier == "M":
1283                 p4_edit(path)
1284                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1285                     filesToChangeExecBit[path] = diff['dst_mode']
1286                 editedFiles.add(path)
1287             elif modifier == "A":
1288                 filesToAdd.add(path)
1289                 filesToChangeExecBit[path] = diff['dst_mode']
1290                 if path in filesToDelete:
1291                     filesToDelete.remove(path)
1292             elif modifier == "D":
1293                 filesToDelete.add(path)
1294                 if path in filesToAdd:
1295                     filesToAdd.remove(path)
1296             elif modifier == "C":
1297                 src, dest = diff['src'], diff['dst']
1298                 p4_integrate(src, dest)
1299                 pureRenameCopy.add(dest)
1300                 if diff['src_sha1'] != diff['dst_sha1']:
1301                     p4_edit(dest)
1302                     pureRenameCopy.discard(dest)
1303                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1304                     p4_edit(dest)
1305                     pureRenameCopy.discard(dest)
1306                     filesToChangeExecBit[dest] = diff['dst_mode']
1307                 if self.isWindows:
1308                     # turn off read-only attribute
1309                     os.chmod(dest, stat.S_IWRITE)
1310                 os.unlink(dest)
1311                 editedFiles.add(dest)
1312             elif modifier == "R":
1313                 src, dest = diff['src'], diff['dst']
1314                 if self.p4HasMoveCommand:
1315                     p4_edit(src)        # src must be open before move
1316                     p4_move(src, dest)  # opens for (move/delete, move/add)
1317                 else:
1318                     p4_integrate(src, dest)
1319                     if diff['src_sha1'] != diff['dst_sha1']:
1320                         p4_edit(dest)
1321                     else:
1322                         pureRenameCopy.add(dest)
1323                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1324                     if not self.p4HasMoveCommand:
1325                         p4_edit(dest)   # with move: already open, writable
1326                     filesToChangeExecBit[dest] = diff['dst_mode']
1327                 if not self.p4HasMoveCommand:
1328                     if self.isWindows:
1329                         os.chmod(dest, stat.S_IWRITE)
1330                     os.unlink(dest)
1331                     filesToDelete.add(src)
1332                 editedFiles.add(dest)
1333             else:
1334                 die("unknown modifier %s for %s" % (modifier, path))
1335
1336         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1337         patchcmd = diffcmd + " | git apply "
1338         tryPatchCmd = patchcmd + "--check -"
1339         applyPatchCmd = patchcmd + "--check --apply -"
1340         patch_succeeded = True
1341
1342         if os.system(tryPatchCmd) != 0:
1343             fixed_rcs_keywords = False
1344             patch_succeeded = False
1345             print "Unfortunately applying the change failed!"
1346
1347             # Patch failed, maybe it's just RCS keyword woes. Look through
1348             # the patch to see if that's possible.
1349             if gitConfigBool("git-p4.attemptRCSCleanup"):
1350                 file = None
1351                 pattern = None
1352                 kwfiles = {}
1353                 for file in editedFiles | filesToDelete:
1354                     # did this file's delta contain RCS keywords?
1355                     pattern = p4_keywords_regexp_for_file(file)
1356
1357                     if pattern:
1358                         # this file is a possibility...look for RCS keywords.
1359                         regexp = re.compile(pattern, re.VERBOSE)
1360                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1361                             if regexp.search(line):
1362                                 if verbose:
1363                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1364                                 kwfiles[file] = pattern
1365                                 break
1366
1367                 for file in kwfiles:
1368                     if verbose:
1369                         print "zapping %s with %s" % (line,pattern)
1370                     # File is being deleted, so not open in p4.  Must
1371                     # disable the read-only bit on windows.
1372                     if self.isWindows and file not in editedFiles:
1373                         os.chmod(file, stat.S_IWRITE)
1374                     self.patchRCSKeywords(file, kwfiles[file])
1375                     fixed_rcs_keywords = True
1376
1377             if fixed_rcs_keywords:
1378                 print "Retrying the patch with RCS keywords cleaned up"
1379                 if os.system(tryPatchCmd) == 0:
1380                     patch_succeeded = True
1381
1382         if not patch_succeeded:
1383             for f in editedFiles:
1384                 p4_revert(f)
1385             return False
1386
1387         #
1388         # Apply the patch for real, and do add/delete/+x handling.
1389         #
1390         system(applyPatchCmd)
1391
1392         for f in filesToAdd:
1393             p4_add(f)
1394         for f in filesToDelete:
1395             p4_revert(f)
1396             p4_delete(f)
1397
1398         # Set/clear executable bits
1399         for f in filesToChangeExecBit.keys():
1400             mode = filesToChangeExecBit[f]
1401             setP4ExecBit(f, mode)
1402
1403         #
1404         # Build p4 change description, starting with the contents
1405         # of the git commit message.
1406         #
1407         logMessage = extractLogMessageFromGitCommit(id)
1408         logMessage = logMessage.strip()
1409         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1410
1411         template = self.prepareSubmitTemplate()
1412         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1413
1414         if self.preserveUser:
1415            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1416
1417         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1418             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1419             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1420             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1421
1422         separatorLine = "######## everything below this line is just the diff #######\n"
1423         if not self.prepare_p4_only:
1424             submitTemplate += separatorLine
1425             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1426
1427         (handle, fileName) = tempfile.mkstemp()
1428         tmpFile = os.fdopen(handle, "w+b")
1429         if self.isWindows:
1430             submitTemplate = submitTemplate.replace("\n", "\r\n")
1431         tmpFile.write(submitTemplate)
1432         tmpFile.close()
1433
1434         if self.prepare_p4_only:
1435             #
1436             # Leave the p4 tree prepared, and the submit template around
1437             # and let the user decide what to do next
1438             #
1439             print
1440             print "P4 workspace prepared for submission."
1441             print "To submit or revert, go to client workspace"
1442             print "  " + self.clientPath
1443             print
1444             print "To submit, use \"p4 submit\" to write a new description,"
1445             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1446                   " \"git p4\"." % fileName
1447             print "You can delete the file \"%s\" when finished." % fileName
1448
1449             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1450                 print "To preserve change ownership by user %s, you must\n" \
1451                       "do \"p4 change -f <change>\" after submitting and\n" \
1452                       "edit the User field."
1453             if pureRenameCopy:
1454                 print "After submitting, renamed files must be re-synced."
1455                 print "Invoke \"p4 sync -f\" on each of these files:"
1456                 for f in pureRenameCopy:
1457                     print "  " + f
1458
1459             print
1460             print "To revert the changes, use \"p4 revert ...\", and delete"
1461             print "the submit template file \"%s\"" % fileName
1462             if filesToAdd:
1463                 print "Since the commit adds new files, they must be deleted:"
1464                 for f in filesToAdd:
1465                     print "  " + f
1466             print
1467             return True
1468
1469         #
1470         # Let the user edit the change description, then submit it.
1471         #
1472         if self.edit_template(fileName):
1473             # read the edited message and submit
1474             ret = True
1475             tmpFile = open(fileName, "rb")
1476             message = tmpFile.read()
1477             tmpFile.close()
1478             if self.isWindows:
1479                 message = message.replace("\r\n", "\n")
1480             submitTemplate = message[:message.index(separatorLine)]
1481             p4_write_pipe(['submit', '-i'], submitTemplate)
1482
1483             if self.preserveUser:
1484                 if p4User:
1485                     # Get last changelist number. Cannot easily get it from
1486                     # the submit command output as the output is
1487                     # unmarshalled.
1488                     changelist = self.lastP4Changelist()
1489                     self.modifyChangelistUser(changelist, p4User)
1490
1491             # The rename/copy happened by applying a patch that created a
1492             # new file.  This leaves it writable, which confuses p4.
1493             for f in pureRenameCopy:
1494                 p4_sync(f, "-f")
1495
1496         else:
1497             # skip this patch
1498             ret = False
1499             print "Submission cancelled, undoing p4 changes."
1500             for f in editedFiles:
1501                 p4_revert(f)
1502             for f in filesToAdd:
1503                 p4_revert(f)
1504                 os.remove(f)
1505             for f in filesToDelete:
1506                 p4_revert(f)
1507
1508         os.remove(fileName)
1509         return ret
1510
1511     # Export git tags as p4 labels. Create a p4 label and then tag
1512     # with that.
1513     def exportGitTags(self, gitTags):
1514         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1515         if len(validLabelRegexp) == 0:
1516             validLabelRegexp = defaultLabelRegexp
1517         m = re.compile(validLabelRegexp)
1518
1519         for name in gitTags:
1520
1521             if not m.match(name):
1522                 if verbose:
1523                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1524                 continue
1525
1526             # Get the p4 commit this corresponds to
1527             logMessage = extractLogMessageFromGitCommit(name)
1528             values = extractSettingsGitLog(logMessage)
1529
1530             if not values.has_key('change'):
1531                 # a tag pointing to something not sent to p4; ignore
1532                 if verbose:
1533                     print "git tag %s does not give a p4 commit" % name
1534                 continue
1535             else:
1536                 changelist = values['change']
1537
1538             # Get the tag details.
1539             inHeader = True
1540             isAnnotated = False
1541             body = []
1542             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1543                 l = l.strip()
1544                 if inHeader:
1545                     if re.match(r'tag\s+', l):
1546                         isAnnotated = True
1547                     elif re.match(r'\s*$', l):
1548                         inHeader = False
1549                         continue
1550                 else:
1551                     body.append(l)
1552
1553             if not isAnnotated:
1554                 body = ["lightweight tag imported by git p4\n"]
1555
1556             # Create the label - use the same view as the client spec we are using
1557             clientSpec = getClientSpec()
1558
1559             labelTemplate  = "Label: %s\n" % name
1560             labelTemplate += "Description:\n"
1561             for b in body:
1562                 labelTemplate += "\t" + b + "\n"
1563             labelTemplate += "View:\n"
1564             for depot_side in clientSpec.mappings:
1565                 labelTemplate += "\t%s\n" % depot_side
1566
1567             if self.dry_run:
1568                 print "Would create p4 label %s for tag" % name
1569             elif self.prepare_p4_only:
1570                 print "Not creating p4 label %s for tag due to option" \
1571                       " --prepare-p4-only" % name
1572             else:
1573                 p4_write_pipe(["label", "-i"], labelTemplate)
1574
1575                 # Use the label
1576                 p4_system(["tag", "-l", name] +
1577                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1578
1579                 if verbose:
1580                     print "created p4 label for tag %s" % name
1581
1582     def run(self, args):
1583         if len(args) == 0:
1584             self.master = currentGitBranch()
1585             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1586                 die("Detecting current git branch failed!")
1587         elif len(args) == 1:
1588             self.master = args[0]
1589             if not branchExists(self.master):
1590                 die("Branch %s does not exist" % self.master)
1591         else:
1592             return False
1593
1594         allowSubmit = gitConfig("git-p4.allowSubmit")
1595         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1596             die("%s is not in git-p4.allowSubmit" % self.master)
1597
1598         [upstream, settings] = findUpstreamBranchPoint()
1599         self.depotPath = settings['depot-paths'][0]
1600         if len(self.origin) == 0:
1601             self.origin = upstream
1602
1603         if self.preserveUser:
1604             if not self.canChangeChangelists():
1605                 die("Cannot preserve user names without p4 super-user or admin permissions")
1606
1607         # if not set from the command line, try the config file
1608         if self.conflict_behavior is None:
1609             val = gitConfig("git-p4.conflict")
1610             if val:
1611                 if val not in self.conflict_behavior_choices:
1612                     die("Invalid value '%s' for config git-p4.conflict" % val)
1613             else:
1614                 val = "ask"
1615             self.conflict_behavior = val
1616
1617         if self.verbose:
1618             print "Origin branch is " + self.origin
1619
1620         if len(self.depotPath) == 0:
1621             print "Internal error: cannot locate perforce depot path from existing branches"
1622             sys.exit(128)
1623
1624         self.useClientSpec = False
1625         if gitConfigBool("git-p4.useclientspec"):
1626             self.useClientSpec = True
1627         if self.useClientSpec:
1628             self.clientSpecDirs = getClientSpec()
1629
1630         if self.useClientSpec:
1631             # all files are relative to the client spec
1632             self.clientPath = getClientRoot()
1633         else:
1634             self.clientPath = p4Where(self.depotPath)
1635
1636         if self.clientPath == "":
1637             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1638
1639         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1640         self.oldWorkingDirectory = os.getcwd()
1641
1642         # ensure the clientPath exists
1643         new_client_dir = False
1644         if not os.path.exists(self.clientPath):
1645             new_client_dir = True
1646             os.makedirs(self.clientPath)
1647
1648         chdir(self.clientPath, is_client_path=True)
1649         if self.dry_run:
1650             print "Would synchronize p4 checkout in %s" % self.clientPath
1651         else:
1652             print "Synchronizing p4 checkout..."
1653             if new_client_dir:
1654                 # old one was destroyed, and maybe nobody told p4
1655                 p4_sync("...", "-f")
1656             else:
1657                 p4_sync("...")
1658         self.check()
1659
1660         commits = []
1661         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1662             commits.append(line.strip())
1663         commits.reverse()
1664
1665         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1666             self.checkAuthorship = False
1667         else:
1668             self.checkAuthorship = True
1669
1670         if self.preserveUser:
1671             self.checkValidP4Users(commits)
1672
1673         #
1674         # Build up a set of options to be passed to diff when
1675         # submitting each commit to p4.
1676         #
1677         if self.detectRenames:
1678             # command-line -M arg
1679             self.diffOpts = "-M"
1680         else:
1681             # If not explicitly set check the config variable
1682             detectRenames = gitConfig("git-p4.detectRenames")
1683
1684             if detectRenames.lower() == "false" or detectRenames == "":
1685                 self.diffOpts = ""
1686             elif detectRenames.lower() == "true":
1687                 self.diffOpts = "-M"
1688             else:
1689                 self.diffOpts = "-M%s" % detectRenames
1690
1691         # no command-line arg for -C or --find-copies-harder, just
1692         # config variables
1693         detectCopies = gitConfig("git-p4.detectCopies")
1694         if detectCopies.lower() == "false" or detectCopies == "":
1695             pass
1696         elif detectCopies.lower() == "true":
1697             self.diffOpts += " -C"
1698         else:
1699             self.diffOpts += " -C%s" % detectCopies
1700
1701         if gitConfigBool("git-p4.detectCopiesHarder"):
1702             self.diffOpts += " --find-copies-harder"
1703
1704         #
1705         # Apply the commits, one at a time.  On failure, ask if should
1706         # continue to try the rest of the patches, or quit.
1707         #
1708         if self.dry_run:
1709             print "Would apply"
1710         applied = []
1711         last = len(commits) - 1
1712         for i, commit in enumerate(commits):
1713             if self.dry_run:
1714                 print " ", read_pipe(["git", "show", "-s",
1715                                       "--format=format:%h %s", commit])
1716                 ok = True
1717             else:
1718                 ok = self.applyCommit(commit)
1719             if ok:
1720                 applied.append(commit)
1721             else:
1722                 if self.prepare_p4_only and i < last:
1723                     print "Processing only the first commit due to option" \
1724                           " --prepare-p4-only"
1725                     break
1726                 if i < last:
1727                     quit = False
1728                     while True:
1729                         # prompt for what to do, or use the option/variable
1730                         if self.conflict_behavior == "ask":
1731                             print "What do you want to do?"
1732                             response = raw_input("[s]kip this commit but apply"
1733                                                  " the rest, or [q]uit? ")
1734                             if not response:
1735                                 continue
1736                         elif self.conflict_behavior == "skip":
1737                             response = "s"
1738                         elif self.conflict_behavior == "quit":
1739                             response = "q"
1740                         else:
1741                             die("Unknown conflict_behavior '%s'" %
1742                                 self.conflict_behavior)
1743
1744                         if response[0] == "s":
1745                             print "Skipping this commit, but applying the rest"
1746                             break
1747                         if response[0] == "q":
1748                             print "Quitting"
1749                             quit = True
1750                             break
1751                     if quit:
1752                         break
1753
1754         chdir(self.oldWorkingDirectory)
1755
1756         if self.dry_run:
1757             pass
1758         elif self.prepare_p4_only:
1759             pass
1760         elif len(commits) == len(applied):
1761             print "All commits applied!"
1762
1763             sync = P4Sync()
1764             if self.branch:
1765                 sync.branch = self.branch
1766             sync.run([])
1767
1768             rebase = P4Rebase()
1769             rebase.rebase()
1770
1771         else:
1772             if len(applied) == 0:
1773                 print "No commits applied."
1774             else:
1775                 print "Applied only the commits marked with '*':"
1776                 for c in commits:
1777                     if c in applied:
1778                         star = "*"
1779                     else:
1780                         star = " "
1781                     print star, read_pipe(["git", "show", "-s",
1782                                            "--format=format:%h %s",  c])
1783                 print "You will have to do 'git p4 sync' and rebase."
1784
1785         if gitConfigBool("git-p4.exportLabels"):
1786             self.exportLabels = True
1787
1788         if self.exportLabels:
1789             p4Labels = getP4Labels(self.depotPath)
1790             gitTags = getGitTags()
1791
1792             missingGitTags = gitTags - p4Labels
1793             self.exportGitTags(missingGitTags)
1794
1795         # exit with error unless everything applied perfectly
1796         if len(commits) != len(applied):
1797                 sys.exit(1)
1798
1799         return True
1800
1801 class View(object):
1802     """Represent a p4 view ("p4 help views"), and map files in a
1803        repo according to the view."""
1804
1805     def __init__(self, client_name):
1806         self.mappings = []
1807         self.client_prefix = "//%s/" % client_name
1808         # cache results of "p4 where" to lookup client file locations
1809         self.client_spec_path_cache = {}
1810
1811     def append(self, view_line):
1812         """Parse a view line, splitting it into depot and client
1813            sides.  Append to self.mappings, preserving order.  This
1814            is only needed for tag creation."""
1815
1816         # Split the view line into exactly two words.  P4 enforces
1817         # structure on these lines that simplifies this quite a bit.
1818         #
1819         # Either or both words may be double-quoted.
1820         # Single quotes do not matter.
1821         # Double-quote marks cannot occur inside the words.
1822         # A + or - prefix is also inside the quotes.
1823         # There are no quotes unless they contain a space.
1824         # The line is already white-space stripped.
1825         # The two words are separated by a single space.
1826         #
1827         if view_line[0] == '"':
1828             # First word is double quoted.  Find its end.
1829             close_quote_index = view_line.find('"', 1)
1830             if close_quote_index <= 0:
1831                 die("No first-word closing quote found: %s" % view_line)
1832             depot_side = view_line[1:close_quote_index]
1833             # skip closing quote and space
1834             rhs_index = close_quote_index + 1 + 1
1835         else:
1836             space_index = view_line.find(" ")
1837             if space_index <= 0:
1838                 die("No word-splitting space found: %s" % view_line)
1839             depot_side = view_line[0:space_index]
1840             rhs_index = space_index + 1
1841
1842         # prefix + means overlay on previous mapping
1843         if depot_side.startswith("+"):
1844             depot_side = depot_side[1:]
1845
1846         # prefix - means exclude this path, leave out of mappings
1847         exclude = False
1848         if depot_side.startswith("-"):
1849             exclude = True
1850             depot_side = depot_side[1:]
1851
1852         if not exclude:
1853             self.mappings.append(depot_side)
1854
1855     def convert_client_path(self, clientFile):
1856         # chop off //client/ part to make it relative
1857         if not clientFile.startswith(self.client_prefix):
1858             die("No prefix '%s' on clientFile '%s'" %
1859                 (self.client_prefix, clientFile))
1860         return clientFile[len(self.client_prefix):]
1861
1862     def update_client_spec_path_cache(self, files):
1863         """ Caching file paths by "p4 where" batch query """
1864
1865         # List depot file paths exclude that already cached
1866         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1867
1868         if len(fileArgs) == 0:
1869             return  # All files in cache
1870
1871         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1872         for res in where_result:
1873             if "code" in res and res["code"] == "error":
1874                 # assume error is "... file(s) not in client view"
1875                 continue
1876             if "clientFile" not in res:
1877                 die("No clientFile in 'p4 where' output")
1878             if "unmap" in res:
1879                 # it will list all of them, but only one not unmap-ped
1880                 continue
1881             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1882
1883         # not found files or unmap files set to ""
1884         for depotFile in fileArgs:
1885             if depotFile not in self.client_spec_path_cache:
1886                 self.client_spec_path_cache[depotFile] = ""
1887
1888     def map_in_client(self, depot_path):
1889         """Return the relative location in the client where this
1890            depot file should live.  Returns "" if the file should
1891            not be mapped in the client."""
1892
1893         if depot_path in self.client_spec_path_cache:
1894             return self.client_spec_path_cache[depot_path]
1895
1896         die( "Error: %s is not found in client spec path" % depot_path )
1897         return ""
1898
1899 class P4Sync(Command, P4UserMap):
1900     delete_actions = ( "delete", "move/delete", "purge" )
1901
1902     def __init__(self):
1903         Command.__init__(self)
1904         P4UserMap.__init__(self)
1905         self.options = [
1906                 optparse.make_option("--branch", dest="branch"),
1907                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1908                 optparse.make_option("--changesfile", dest="changesFile"),
1909                 optparse.make_option("--silent", dest="silent", action="store_true"),
1910                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1911                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1912                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1913                                      help="Import into refs/heads/ , not refs/remotes"),
1914                 optparse.make_option("--max-changes", dest="maxChanges"),
1915                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1916                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1917                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1918                                      help="Only sync files that are included in the Perforce Client Spec"),
1919                 optparse.make_option("-/", dest="cloneExclude",
1920                                      action="append", type="string",
1921                                      help="exclude depot path"),
1922         ]
1923         self.description = """Imports from Perforce into a git repository.\n
1924     example:
1925     //depot/my/project/ -- to import the current head
1926     //depot/my/project/@all -- to import everything
1927     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1928
1929     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1930
1931         self.usage += " //depot/path[@revRange]"
1932         self.silent = False
1933         self.createdBranches = set()
1934         self.committedChanges = set()
1935         self.branch = ""
1936         self.detectBranches = False
1937         self.detectLabels = False
1938         self.importLabels = False
1939         self.changesFile = ""
1940         self.syncWithOrigin = True
1941         self.importIntoRemotes = True
1942         self.maxChanges = ""
1943         self.keepRepoPath = False
1944         self.depotPaths = None
1945         self.p4BranchesInGit = []
1946         self.cloneExclude = []
1947         self.useClientSpec = False
1948         self.useClientSpec_from_options = False
1949         self.clientSpecDirs = None
1950         self.tempBranches = []
1951         self.tempBranchLocation = "git-p4-tmp"
1952
1953         if gitConfig("git-p4.syncFromOrigin") == "false":
1954             self.syncWithOrigin = False
1955
1956     # This is required for the "append" cloneExclude action
1957     def ensure_value(self, attr, value):
1958         if not hasattr(self, attr) or getattr(self, attr) is None:
1959             setattr(self, attr, value)
1960         return getattr(self, attr)
1961
1962     # Force a checkpoint in fast-import and wait for it to finish
1963     def checkpoint(self):
1964         self.gitStream.write("checkpoint\n\n")
1965         self.gitStream.write("progress checkpoint\n\n")
1966         out = self.gitOutput.readline()
1967         if self.verbose:
1968             print "checkpoint finished: " + out
1969
1970     def extractFilesFromCommit(self, commit):
1971         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1972                              for path in self.cloneExclude]
1973         files = []
1974         fnum = 0
1975         while commit.has_key("depotFile%s" % fnum):
1976             path =  commit["depotFile%s" % fnum]
1977
1978             if [p for p in self.cloneExclude
1979                 if p4PathStartsWith(path, p)]:
1980                 found = False
1981             else:
1982                 found = [p for p in self.depotPaths
1983                          if p4PathStartsWith(path, p)]
1984             if not found:
1985                 fnum = fnum + 1
1986                 continue
1987
1988             file = {}
1989             file["path"] = path
1990             file["rev"] = commit["rev%s" % fnum]
1991             file["action"] = commit["action%s" % fnum]
1992             file["type"] = commit["type%s" % fnum]
1993             files.append(file)
1994             fnum = fnum + 1
1995         return files
1996
1997     def stripRepoPath(self, path, prefixes):
1998         """When streaming files, this is called to map a p4 depot path
1999            to where it should go in git.  The prefixes are either
2000            self.depotPaths, or self.branchPrefixes in the case of
2001            branch detection."""
2002
2003         if self.useClientSpec:
2004             # branch detection moves files up a level (the branch name)
2005             # from what client spec interpretation gives
2006             path = self.clientSpecDirs.map_in_client(path)
2007             if self.detectBranches:
2008                 for b in self.knownBranches:
2009                     if path.startswith(b + "/"):
2010                         path = path[len(b)+1:]
2011
2012         elif self.keepRepoPath:
2013             # Preserve everything in relative path name except leading
2014             # //depot/; just look at first prefix as they all should
2015             # be in the same depot.
2016             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2017             if p4PathStartsWith(path, depot):
2018                 path = path[len(depot):]
2019
2020         else:
2021             for p in prefixes:
2022                 if p4PathStartsWith(path, p):
2023                     path = path[len(p):]
2024                     break
2025
2026         path = wildcard_decode(path)
2027         return path
2028
2029     def splitFilesIntoBranches(self, commit):
2030         """Look at each depotFile in the commit to figure out to what
2031            branch it belongs."""
2032
2033         if self.clientSpecDirs:
2034             files = self.extractFilesFromCommit(commit)
2035             self.clientSpecDirs.update_client_spec_path_cache(files)
2036
2037         branches = {}
2038         fnum = 0
2039         while commit.has_key("depotFile%s" % fnum):
2040             path =  commit["depotFile%s" % fnum]
2041             found = [p for p in self.depotPaths
2042                      if p4PathStartsWith(path, p)]
2043             if not found:
2044                 fnum = fnum + 1
2045                 continue
2046
2047             file = {}
2048             file["path"] = path
2049             file["rev"] = commit["rev%s" % fnum]
2050             file["action"] = commit["action%s" % fnum]
2051             file["type"] = commit["type%s" % fnum]
2052             fnum = fnum + 1
2053
2054             # start with the full relative path where this file would
2055             # go in a p4 client
2056             if self.useClientSpec:
2057                 relPath = self.clientSpecDirs.map_in_client(path)
2058             else:
2059                 relPath = self.stripRepoPath(path, self.depotPaths)
2060
2061             for branch in self.knownBranches.keys():
2062                 # add a trailing slash so that a commit into qt/4.2foo
2063                 # doesn't end up in qt/4.2, e.g.
2064                 if relPath.startswith(branch + "/"):
2065                     if branch not in branches:
2066                         branches[branch] = []
2067                     branches[branch].append(file)
2068                     break
2069
2070         return branches
2071
2072     # output one file from the P4 stream
2073     # - helper for streamP4Files
2074
2075     def streamOneP4File(self, file, contents):
2076         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2077         if verbose:
2078             sys.stderr.write("%s\n" % relPath)
2079
2080         (type_base, type_mods) = split_p4_type(file["type"])
2081
2082         git_mode = "100644"
2083         if "x" in type_mods:
2084             git_mode = "100755"
2085         if type_base == "symlink":
2086             git_mode = "120000"
2087             # p4 print on a symlink sometimes contains "target\n";
2088             # if it does, remove the newline
2089             data = ''.join(contents)
2090             if not data:
2091                 # Some version of p4 allowed creating a symlink that pointed
2092                 # to nothing.  This causes p4 errors when checking out such
2093                 # a change, and errors here too.  Work around it by ignoring
2094                 # the bad symlink; hopefully a future change fixes it.
2095                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2096                 return
2097             elif data[-1] == '\n':
2098                 contents = [data[:-1]]
2099             else:
2100                 contents = [data]
2101
2102         if type_base == "utf16":
2103             # p4 delivers different text in the python output to -G
2104             # than it does when using "print -o", or normal p4 client
2105             # operations.  utf16 is converted to ascii or utf8, perhaps.
2106             # But ascii text saved as -t utf16 is completely mangled.
2107             # Invoke print -o to get the real contents.
2108             #
2109             # On windows, the newlines will always be mangled by print, so put
2110             # them back too.  This is not needed to the cygwin windows version,
2111             # just the native "NT" type.
2112             #
2113             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2114             if p4_version_string().find("/NT") >= 0:
2115                 text = text.replace("\r\n", "\n")
2116             contents = [ text ]
2117
2118         if type_base == "apple":
2119             # Apple filetype files will be streamed as a concatenation of
2120             # its appledouble header and the contents.  This is useless
2121             # on both macs and non-macs.  If using "print -q -o xx", it
2122             # will create "xx" with the data, and "%xx" with the header.
2123             # This is also not very useful.
2124             #
2125             # Ideally, someday, this script can learn how to generate
2126             # appledouble files directly and import those to git, but
2127             # non-mac machines can never find a use for apple filetype.
2128             print "\nIgnoring apple filetype file %s" % file['depotFile']
2129             return
2130
2131         # Note that we do not try to de-mangle keywords on utf16 files,
2132         # even though in theory somebody may want that.
2133         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2134         if pattern:
2135             regexp = re.compile(pattern, re.VERBOSE)
2136             text = ''.join(contents)
2137             text = regexp.sub(r'$\1$', text)
2138             contents = [ text ]
2139
2140         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2141
2142         # total length...
2143         length = 0
2144         for d in contents:
2145             length = length + len(d)
2146
2147         self.gitStream.write("data %d\n" % length)
2148         for d in contents:
2149             self.gitStream.write(d)
2150         self.gitStream.write("\n")
2151
2152     def streamOneP4Deletion(self, file):
2153         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2154         if verbose:
2155             sys.stderr.write("delete %s\n" % relPath)
2156         self.gitStream.write("D %s\n" % relPath)
2157
2158     # handle another chunk of streaming data
2159     def streamP4FilesCb(self, marshalled):
2160
2161         # catch p4 errors and complain
2162         err = None
2163         if "code" in marshalled:
2164             if marshalled["code"] == "error":
2165                 if "data" in marshalled:
2166                     err = marshalled["data"].rstrip()
2167         if err:
2168             f = None
2169             if self.stream_have_file_info:
2170                 if "depotFile" in self.stream_file:
2171                     f = self.stream_file["depotFile"]
2172             # force a failure in fast-import, else an empty
2173             # commit will be made
2174             self.gitStream.write("\n")
2175             self.gitStream.write("die-now\n")
2176             self.gitStream.close()
2177             # ignore errors, but make sure it exits first
2178             self.importProcess.wait()
2179             if f:
2180                 die("Error from p4 print for %s: %s" % (f, err))
2181             else:
2182                 die("Error from p4 print: %s" % err)
2183
2184         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2185             # start of a new file - output the old one first
2186             self.streamOneP4File(self.stream_file, self.stream_contents)
2187             self.stream_file = {}
2188             self.stream_contents = []
2189             self.stream_have_file_info = False
2190
2191         # pick up the new file information... for the
2192         # 'data' field we need to append to our array
2193         for k in marshalled.keys():
2194             if k == 'data':
2195                 self.stream_contents.append(marshalled['data'])
2196             else:
2197                 self.stream_file[k] = marshalled[k]
2198
2199         self.stream_have_file_info = True
2200
2201     # Stream directly from "p4 files" into "git fast-import"
2202     def streamP4Files(self, files):
2203         filesForCommit = []
2204         filesToRead = []
2205         filesToDelete = []
2206
2207         for f in files:
2208             # if using a client spec, only add the files that have
2209             # a path in the client
2210             if self.clientSpecDirs:
2211                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2212                     continue
2213
2214             filesForCommit.append(f)
2215             if f['action'] in self.delete_actions:
2216                 filesToDelete.append(f)
2217             else:
2218                 filesToRead.append(f)
2219
2220         # deleted files...
2221         for f in filesToDelete:
2222             self.streamOneP4Deletion(f)
2223
2224         if len(filesToRead) > 0:
2225             self.stream_file = {}
2226             self.stream_contents = []
2227             self.stream_have_file_info = False
2228
2229             # curry self argument
2230             def streamP4FilesCbSelf(entry):
2231                 self.streamP4FilesCb(entry)
2232
2233             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2234
2235             p4CmdList(["-x", "-", "print"],
2236                       stdin=fileArgs,
2237                       cb=streamP4FilesCbSelf)
2238
2239             # do the last chunk
2240             if self.stream_file.has_key('depotFile'):
2241                 self.streamOneP4File(self.stream_file, self.stream_contents)
2242
2243     def make_email(self, userid):
2244         if userid in self.users:
2245             return self.users[userid]
2246         else:
2247             return "%s <a@b>" % userid
2248
2249     # Stream a p4 tag
2250     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2251         if verbose:
2252             print "writing tag %s for commit %s" % (labelName, commit)
2253         gitStream.write("tag %s\n" % labelName)
2254         gitStream.write("from %s\n" % commit)
2255
2256         if labelDetails.has_key('Owner'):
2257             owner = labelDetails["Owner"]
2258         else:
2259             owner = None
2260
2261         # Try to use the owner of the p4 label, or failing that,
2262         # the current p4 user id.
2263         if owner:
2264             email = self.make_email(owner)
2265         else:
2266             email = self.make_email(self.p4UserId())
2267         tagger = "%s %s %s" % (email, epoch, self.tz)
2268
2269         gitStream.write("tagger %s\n" % tagger)
2270
2271         print "labelDetails=",labelDetails
2272         if labelDetails.has_key('Description'):
2273             description = labelDetails['Description']
2274         else:
2275             description = 'Label from git p4'
2276
2277         gitStream.write("data %d\n" % len(description))
2278         gitStream.write(description)
2279         gitStream.write("\n")
2280
2281     def commit(self, details, files, branch, parent = ""):
2282         epoch = details["time"]
2283         author = details["user"]
2284
2285         if self.verbose:
2286             print "commit into %s" % branch
2287
2288         # start with reading files; if that fails, we should not
2289         # create a commit.
2290         new_files = []
2291         for f in files:
2292             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2293                 new_files.append (f)
2294             else:
2295                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2296
2297         if self.clientSpecDirs:
2298             self.clientSpecDirs.update_client_spec_path_cache(files)
2299
2300         self.gitStream.write("commit %s\n" % branch)
2301 #        gitStream.write("mark :%s\n" % details["change"])
2302         self.committedChanges.add(int(details["change"]))
2303         committer = ""
2304         if author not in self.users:
2305             self.getUserMapFromPerforceServer()
2306         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2307
2308         self.gitStream.write("committer %s\n" % committer)
2309
2310         self.gitStream.write("data <<EOT\n")
2311         self.gitStream.write(details["desc"])
2312         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2313                              (','.join(self.branchPrefixes), details["change"]))
2314         if len(details['options']) > 0:
2315             self.gitStream.write(": options = %s" % details['options'])
2316         self.gitStream.write("]\nEOT\n\n")
2317
2318         if len(parent) > 0:
2319             if self.verbose:
2320                 print "parent %s" % parent
2321             self.gitStream.write("from %s\n" % parent)
2322
2323         self.streamP4Files(new_files)
2324         self.gitStream.write("\n")
2325
2326         change = int(details["change"])
2327
2328         if self.labels.has_key(change):
2329             label = self.labels[change]
2330             labelDetails = label[0]
2331             labelRevisions = label[1]
2332             if self.verbose:
2333                 print "Change %s is labelled %s" % (change, labelDetails)
2334
2335             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2336                                                 for p in self.branchPrefixes])
2337
2338             if len(files) == len(labelRevisions):
2339
2340                 cleanedFiles = {}
2341                 for info in files:
2342                     if info["action"] in self.delete_actions:
2343                         continue
2344                     cleanedFiles[info["depotFile"]] = info["rev"]
2345
2346                 if cleanedFiles == labelRevisions:
2347                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2348
2349                 else:
2350                     if not self.silent:
2351                         print ("Tag %s does not match with change %s: files do not match."
2352                                % (labelDetails["label"], change))
2353
2354             else:
2355                 if not self.silent:
2356                     print ("Tag %s does not match with change %s: file count is different."
2357                            % (labelDetails["label"], change))
2358
2359     # Build a dictionary of changelists and labels, for "detect-labels" option.
2360     def getLabels(self):
2361         self.labels = {}
2362
2363         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2364         if len(l) > 0 and not self.silent:
2365             print "Finding files belonging to labels in %s" % `self.depotPaths`
2366
2367         for output in l:
2368             label = output["label"]
2369             revisions = {}
2370             newestChange = 0
2371             if self.verbose:
2372                 print "Querying files for label %s" % label
2373             for file in p4CmdList(["files"] +
2374                                       ["%s...@%s" % (p, label)
2375                                           for p in self.depotPaths]):
2376                 revisions[file["depotFile"]] = file["rev"]
2377                 change = int(file["change"])
2378                 if change > newestChange:
2379                     newestChange = change
2380
2381             self.labels[newestChange] = [output, revisions]
2382
2383         if self.verbose:
2384             print "Label changes: %s" % self.labels.keys()
2385
2386     # Import p4 labels as git tags. A direct mapping does not
2387     # exist, so assume that if all the files are at the same revision
2388     # then we can use that, or it's something more complicated we should
2389     # just ignore.
2390     def importP4Labels(self, stream, p4Labels):
2391         if verbose:
2392             print "import p4 labels: " + ' '.join(p4Labels)
2393
2394         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2395         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2396         if len(validLabelRegexp) == 0:
2397             validLabelRegexp = defaultLabelRegexp
2398         m = re.compile(validLabelRegexp)
2399
2400         for name in p4Labels:
2401             commitFound = False
2402
2403             if not m.match(name):
2404                 if verbose:
2405                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2406                 continue
2407
2408             if name in ignoredP4Labels:
2409                 continue
2410
2411             labelDetails = p4CmdList(['label', "-o", name])[0]
2412
2413             # get the most recent changelist for each file in this label
2414             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2415                                 for p in self.depotPaths])
2416
2417             if change.has_key('change'):
2418                 # find the corresponding git commit; take the oldest commit
2419                 changelist = int(change['change'])
2420                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2421                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2422                 if len(gitCommit) == 0:
2423                     print "could not find git commit for changelist %d" % changelist
2424                 else:
2425                     gitCommit = gitCommit.strip()
2426                     commitFound = True
2427                     # Convert from p4 time format
2428                     try:
2429                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2430                     except ValueError:
2431                         print "Could not convert label time %s" % labelDetails['Update']
2432                         tmwhen = 1
2433
2434                     when = int(time.mktime(tmwhen))
2435                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2436                     if verbose:
2437                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2438             else:
2439                 if verbose:
2440                     print "Label %s has no changelists - possibly deleted?" % name
2441
2442             if not commitFound:
2443                 # We can't import this label; don't try again as it will get very
2444                 # expensive repeatedly fetching all the files for labels that will
2445                 # never be imported. If the label is moved in the future, the
2446                 # ignore will need to be removed manually.
2447                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2448
2449     def guessProjectName(self):
2450         for p in self.depotPaths:
2451             if p.endswith("/"):
2452                 p = p[:-1]
2453             p = p[p.strip().rfind("/") + 1:]
2454             if not p.endswith("/"):
2455                p += "/"
2456             return p
2457
2458     def getBranchMapping(self):
2459         lostAndFoundBranches = set()
2460
2461         user = gitConfig("git-p4.branchUser")
2462         if len(user) > 0:
2463             command = "branches -u %s" % user
2464         else:
2465             command = "branches"
2466
2467         for info in p4CmdList(command):
2468             details = p4Cmd(["branch", "-o", info["branch"]])
2469             viewIdx = 0
2470             while details.has_key("View%s" % viewIdx):
2471                 paths = details["View%s" % viewIdx].split(" ")
2472                 viewIdx = viewIdx + 1
2473                 # require standard //depot/foo/... //depot/bar/... mapping
2474                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2475                     continue
2476                 source = paths[0]
2477                 destination = paths[1]
2478                 ## HACK
2479                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2480                     source = source[len(self.depotPaths[0]):-4]
2481                     destination = destination[len(self.depotPaths[0]):-4]
2482
2483                     if destination in self.knownBranches:
2484                         if not self.silent:
2485                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2486                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2487                         continue
2488
2489                     self.knownBranches[destination] = source
2490
2491                     lostAndFoundBranches.discard(destination)
2492
2493                     if source not in self.knownBranches:
2494                         lostAndFoundBranches.add(source)
2495
2496         # Perforce does not strictly require branches to be defined, so we also
2497         # check git config for a branch list.
2498         #
2499         # Example of branch definition in git config file:
2500         # [git-p4]
2501         #   branchList=main:branchA
2502         #   branchList=main:branchB
2503         #   branchList=branchA:branchC
2504         configBranches = gitConfigList("git-p4.branchList")
2505         for branch in configBranches:
2506             if branch:
2507                 (source, destination) = branch.split(":")
2508                 self.knownBranches[destination] = source
2509
2510                 lostAndFoundBranches.discard(destination)
2511
2512                 if source not in self.knownBranches:
2513                     lostAndFoundBranches.add(source)
2514
2515
2516         for branch in lostAndFoundBranches:
2517             self.knownBranches[branch] = branch
2518
2519     def getBranchMappingFromGitBranches(self):
2520         branches = p4BranchesInGit(self.importIntoRemotes)
2521         for branch in branches.keys():
2522             if branch == "master":
2523                 branch = "main"
2524             else:
2525                 branch = branch[len(self.projectName):]
2526             self.knownBranches[branch] = branch
2527
2528     def updateOptionDict(self, d):
2529         option_keys = {}
2530         if self.keepRepoPath:
2531             option_keys['keepRepoPath'] = 1
2532
2533         d["options"] = ' '.join(sorted(option_keys.keys()))
2534
2535     def readOptions(self, d):
2536         self.keepRepoPath = (d.has_key('options')
2537                              and ('keepRepoPath' in d['options']))
2538
2539     def gitRefForBranch(self, branch):
2540         if branch == "main":
2541             return self.refPrefix + "master"
2542
2543         if len(branch) <= 0:
2544             return branch
2545
2546         return self.refPrefix + self.projectName + branch
2547
2548     def gitCommitByP4Change(self, ref, change):
2549         if self.verbose:
2550             print "looking in ref " + ref + " for change %s using bisect..." % change
2551
2552         earliestCommit = ""
2553         latestCommit = parseRevision(ref)
2554
2555         while True:
2556             if self.verbose:
2557                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2558             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2559             if len(next) == 0:
2560                 if self.verbose:
2561                     print "argh"
2562                 return ""
2563             log = extractLogMessageFromGitCommit(next)
2564             settings = extractSettingsGitLog(log)
2565             currentChange = int(settings['change'])
2566             if self.verbose:
2567                 print "current change %s" % currentChange
2568
2569             if currentChange == change:
2570                 if self.verbose:
2571                     print "found %s" % next
2572                 return next
2573
2574             if currentChange < change:
2575                 earliestCommit = "^%s" % next
2576             else:
2577                 latestCommit = "%s" % next
2578
2579         return ""
2580
2581     def importNewBranch(self, branch, maxChange):
2582         # make fast-import flush all changes to disk and update the refs using the checkpoint
2583         # command so that we can try to find the branch parent in the git history
2584         self.gitStream.write("checkpoint\n\n");
2585         self.gitStream.flush();
2586         branchPrefix = self.depotPaths[0] + branch + "/"
2587         range = "@1,%s" % maxChange
2588         #print "prefix" + branchPrefix
2589         changes = p4ChangesForPaths([branchPrefix], range)
2590         if len(changes) <= 0:
2591             return False
2592         firstChange = changes[0]
2593         #print "first change in branch: %s" % firstChange
2594         sourceBranch = self.knownBranches[branch]
2595         sourceDepotPath = self.depotPaths[0] + sourceBranch
2596         sourceRef = self.gitRefForBranch(sourceBranch)
2597         #print "source " + sourceBranch
2598
2599         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2600         #print "branch parent: %s" % branchParentChange
2601         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2602         if len(gitParent) > 0:
2603             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2604             #print "parent git commit: %s" % gitParent
2605
2606         self.importChanges(changes)
2607         return True
2608
2609     def searchParent(self, parent, branch, target):
2610         parentFound = False
2611         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2612                                      "--no-merges", parent]):
2613             blob = blob.strip()
2614             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2615                 parentFound = True
2616                 if self.verbose:
2617                     print "Found parent of %s in commit %s" % (branch, blob)
2618                 break
2619         if parentFound:
2620             return blob
2621         else:
2622             return None
2623
2624     def importChanges(self, changes):
2625         cnt = 1
2626         for change in changes:
2627             description = p4_describe(change)
2628             self.updateOptionDict(description)
2629
2630             if not self.silent:
2631                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2632                 sys.stdout.flush()
2633             cnt = cnt + 1
2634
2635             try:
2636                 if self.detectBranches:
2637                     branches = self.splitFilesIntoBranches(description)
2638                     for branch in branches.keys():
2639                         ## HACK  --hwn
2640                         branchPrefix = self.depotPaths[0] + branch + "/"
2641                         self.branchPrefixes = [ branchPrefix ]
2642
2643                         parent = ""
2644
2645                         filesForCommit = branches[branch]
2646
2647                         if self.verbose:
2648                             print "branch is %s" % branch
2649
2650                         self.updatedBranches.add(branch)
2651
2652                         if branch not in self.createdBranches:
2653                             self.createdBranches.add(branch)
2654                             parent = self.knownBranches[branch]
2655                             if parent == branch:
2656                                 parent = ""
2657                             else:
2658                                 fullBranch = self.projectName + branch
2659                                 if fullBranch not in self.p4BranchesInGit:
2660                                     if not self.silent:
2661                                         print("\n    Importing new branch %s" % fullBranch);
2662                                     if self.importNewBranch(branch, change - 1):
2663                                         parent = ""
2664                                         self.p4BranchesInGit.append(fullBranch)
2665                                     if not self.silent:
2666                                         print("\n    Resuming with change %s" % change);
2667
2668                                 if self.verbose:
2669                                     print "parent determined through known branches: %s" % parent
2670
2671                         branch = self.gitRefForBranch(branch)
2672                         parent = self.gitRefForBranch(parent)
2673
2674                         if self.verbose:
2675                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2676
2677                         if len(parent) == 0 and branch in self.initialParents:
2678                             parent = self.initialParents[branch]
2679                             del self.initialParents[branch]
2680
2681                         blob = None
2682                         if len(parent) > 0:
2683                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2684                             if self.verbose:
2685                                 print "Creating temporary branch: " + tempBranch
2686                             self.commit(description, filesForCommit, tempBranch)
2687                             self.tempBranches.append(tempBranch)
2688                             self.checkpoint()
2689                             blob = self.searchParent(parent, branch, tempBranch)
2690                         if blob:
2691                             self.commit(description, filesForCommit, branch, blob)
2692                         else:
2693                             if self.verbose:
2694                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2695                             self.commit(description, filesForCommit, branch, parent)
2696                 else:
2697                     files = self.extractFilesFromCommit(description)
2698                     self.commit(description, files, self.branch,
2699                                 self.initialParent)
2700                     # only needed once, to connect to the previous commit
2701                     self.initialParent = ""
2702             except IOError:
2703                 print self.gitError.read()
2704                 sys.exit(1)
2705
2706     def importHeadRevision(self, revision):
2707         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2708
2709         details = {}
2710         details["user"] = "git perforce import user"
2711         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2712                            % (' '.join(self.depotPaths), revision))
2713         details["change"] = revision
2714         newestRevision = 0
2715
2716         fileCnt = 0
2717         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2718
2719         for info in p4CmdList(["files"] + fileArgs):
2720
2721             if 'code' in info and info['code'] == 'error':
2722                 sys.stderr.write("p4 returned an error: %s\n"
2723                                  % info['data'])
2724                 if info['data'].find("must refer to client") >= 0:
2725                     sys.stderr.write("This particular p4 error is misleading.\n")
2726                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2727                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2728                 sys.exit(1)
2729             if 'p4ExitCode' in info:
2730                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2731                 sys.exit(1)
2732
2733
2734             change = int(info["change"])
2735             if change > newestRevision:
2736                 newestRevision = change
2737
2738             if info["action"] in self.delete_actions:
2739                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2740                 #fileCnt = fileCnt + 1
2741                 continue
2742
2743             for prop in ["depotFile", "rev", "action", "type" ]:
2744                 details["%s%s" % (prop, fileCnt)] = info[prop]
2745
2746             fileCnt = fileCnt + 1
2747
2748         details["change"] = newestRevision
2749
2750         # Use time from top-most change so that all git p4 clones of
2751         # the same p4 repo have the same commit SHA1s.
2752         res = p4_describe(newestRevision)
2753         details["time"] = res["time"]
2754
2755         self.updateOptionDict(details)
2756         try:
2757             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2758         except IOError:
2759             print "IO error with git fast-import. Is your git version recent enough?"
2760             print self.gitError.read()
2761
2762
2763     def run(self, args):
2764         self.depotPaths = []
2765         self.changeRange = ""
2766         self.previousDepotPaths = []
2767         self.hasOrigin = False
2768
2769         # map from branch depot path to parent branch
2770         self.knownBranches = {}
2771         self.initialParents = {}
2772
2773         if self.importIntoRemotes:
2774             self.refPrefix = "refs/remotes/p4/"
2775         else:
2776             self.refPrefix = "refs/heads/p4/"
2777
2778         if self.syncWithOrigin:
2779             self.hasOrigin = originP4BranchesExist()
2780             if self.hasOrigin:
2781                 if not self.silent:
2782                     print 'Syncing with origin first, using "git fetch origin"'
2783                 system("git fetch origin")
2784
2785         branch_arg_given = bool(self.branch)
2786         if len(self.branch) == 0:
2787             self.branch = self.refPrefix + "master"
2788             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2789                 system("git update-ref %s refs/heads/p4" % self.branch)
2790                 system("git branch -D p4")
2791
2792         # accept either the command-line option, or the configuration variable
2793         if self.useClientSpec:
2794             # will use this after clone to set the variable
2795             self.useClientSpec_from_options = True
2796         else:
2797             if gitConfigBool("git-p4.useclientspec"):
2798                 self.useClientSpec = True
2799         if self.useClientSpec:
2800             self.clientSpecDirs = getClientSpec()
2801
2802         # TODO: should always look at previous commits,
2803         # merge with previous imports, if possible.
2804         if args == []:
2805             if self.hasOrigin:
2806                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2807
2808             # branches holds mapping from branch name to sha1
2809             branches = p4BranchesInGit(self.importIntoRemotes)
2810
2811             # restrict to just this one, disabling detect-branches
2812             if branch_arg_given:
2813                 short = self.branch.split("/")[-1]
2814                 if short in branches:
2815                     self.p4BranchesInGit = [ short ]
2816             else:
2817                 self.p4BranchesInGit = branches.keys()
2818
2819             if len(self.p4BranchesInGit) > 1:
2820                 if not self.silent:
2821                     print "Importing from/into multiple branches"
2822                 self.detectBranches = True
2823                 for branch in branches.keys():
2824                     self.initialParents[self.refPrefix + branch] = \
2825                         branches[branch]
2826
2827             if self.verbose:
2828                 print "branches: %s" % self.p4BranchesInGit
2829
2830             p4Change = 0
2831             for branch in self.p4BranchesInGit:
2832                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2833
2834                 settings = extractSettingsGitLog(logMsg)
2835
2836                 self.readOptions(settings)
2837                 if (settings.has_key('depot-paths')
2838                     and settings.has_key ('change')):
2839                     change = int(settings['change']) + 1
2840                     p4Change = max(p4Change, change)
2841
2842                     depotPaths = sorted(settings['depot-paths'])
2843                     if self.previousDepotPaths == []:
2844                         self.previousDepotPaths = depotPaths
2845                     else:
2846                         paths = []
2847                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2848                             prev_list = prev.split("/")
2849                             cur_list = cur.split("/")
2850                             for i in range(0, min(len(cur_list), len(prev_list))):
2851                                 if cur_list[i] <> prev_list[i]:
2852                                     i = i - 1
2853                                     break
2854
2855                             paths.append ("/".join(cur_list[:i + 1]))
2856
2857                         self.previousDepotPaths = paths
2858
2859             if p4Change > 0:
2860                 self.depotPaths = sorted(self.previousDepotPaths)
2861                 self.changeRange = "@%s,#head" % p4Change
2862                 if not self.silent and not self.detectBranches:
2863                     print "Performing incremental import into %s git branch" % self.branch
2864
2865         # accept multiple ref name abbreviations:
2866         #    refs/foo/bar/branch -> use it exactly
2867         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2868         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2869         if not self.branch.startswith("refs/"):
2870             if self.importIntoRemotes:
2871                 prepend = "refs/remotes/"
2872             else:
2873                 prepend = "refs/heads/"
2874             if not self.branch.startswith("p4/"):
2875                 prepend += "p4/"
2876             self.branch = prepend + self.branch
2877
2878         if len(args) == 0 and self.depotPaths:
2879             if not self.silent:
2880                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2881         else:
2882             if self.depotPaths and self.depotPaths != args:
2883                 print ("previous import used depot path %s and now %s was specified. "
2884                        "This doesn't work!" % (' '.join (self.depotPaths),
2885                                                ' '.join (args)))
2886                 sys.exit(1)
2887
2888             self.depotPaths = sorted(args)
2889
2890         revision = ""
2891         self.users = {}
2892
2893         # Make sure no revision specifiers are used when --changesfile
2894         # is specified.
2895         bad_changesfile = False
2896         if len(self.changesFile) > 0:
2897             for p in self.depotPaths:
2898                 if p.find("@") >= 0 or p.find("#") >= 0:
2899                     bad_changesfile = True
2900                     break
2901         if bad_changesfile:
2902             die("Option --changesfile is incompatible with revision specifiers")
2903
2904         newPaths = []
2905         for p in self.depotPaths:
2906             if p.find("@") != -1:
2907                 atIdx = p.index("@")
2908                 self.changeRange = p[atIdx:]
2909                 if self.changeRange == "@all":
2910                     self.changeRange = ""
2911                 elif ',' not in self.changeRange:
2912                     revision = self.changeRange
2913                     self.changeRange = ""
2914                 p = p[:atIdx]
2915             elif p.find("#") != -1:
2916                 hashIdx = p.index("#")
2917                 revision = p[hashIdx:]
2918                 p = p[:hashIdx]
2919             elif self.previousDepotPaths == []:
2920                 # pay attention to changesfile, if given, else import
2921                 # the entire p4 tree at the head revision
2922                 if len(self.changesFile) == 0:
2923                     revision = "#head"
2924
2925             p = re.sub ("\.\.\.$", "", p)
2926             if not p.endswith("/"):
2927                 p += "/"
2928
2929             newPaths.append(p)
2930
2931         self.depotPaths = newPaths
2932
2933         # --detect-branches may change this for each branch
2934         self.branchPrefixes = self.depotPaths
2935
2936         self.loadUserMapFromCache()
2937         self.labels = {}
2938         if self.detectLabels:
2939             self.getLabels();
2940
2941         if self.detectBranches:
2942             ## FIXME - what's a P4 projectName ?
2943             self.projectName = self.guessProjectName()
2944
2945             if self.hasOrigin:
2946                 self.getBranchMappingFromGitBranches()
2947             else:
2948                 self.getBranchMapping()
2949             if self.verbose:
2950                 print "p4-git branches: %s" % self.p4BranchesInGit
2951                 print "initial parents: %s" % self.initialParents
2952             for b in self.p4BranchesInGit:
2953                 if b != "master":
2954
2955                     ## FIXME
2956                     b = b[len(self.projectName):]
2957                 self.createdBranches.add(b)
2958
2959         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2960
2961         self.importProcess = subprocess.Popen(["git", "fast-import"],
2962                                               stdin=subprocess.PIPE,
2963                                               stdout=subprocess.PIPE,
2964                                               stderr=subprocess.PIPE);
2965         self.gitOutput = self.importProcess.stdout
2966         self.gitStream = self.importProcess.stdin
2967         self.gitError = self.importProcess.stderr
2968
2969         if revision:
2970             self.importHeadRevision(revision)
2971         else:
2972             changes = []
2973
2974             if len(self.changesFile) > 0:
2975                 output = open(self.changesFile).readlines()
2976                 changeSet = set()
2977                 for line in output:
2978                     changeSet.add(int(line))
2979
2980                 for change in changeSet:
2981                     changes.append(change)
2982
2983                 changes.sort()
2984             else:
2985                 # catch "git p4 sync" with no new branches, in a repo that
2986                 # does not have any existing p4 branches
2987                 if len(args) == 0:
2988                     if not self.p4BranchesInGit:
2989                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
2990
2991                     # The default branch is master, unless --branch is used to
2992                     # specify something else.  Make sure it exists, or complain
2993                     # nicely about how to use --branch.
2994                     if not self.detectBranches:
2995                         if not branch_exists(self.branch):
2996                             if branch_arg_given:
2997                                 die("Error: branch %s does not exist." % self.branch)
2998                             else:
2999                                 die("Error: no branch %s; perhaps specify one with --branch." %
3000                                     self.branch)
3001
3002                 if self.verbose:
3003                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3004                                                               self.changeRange)
3005                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
3006
3007                 if len(self.maxChanges) > 0:
3008                     changes = changes[:min(int(self.maxChanges), len(changes))]
3009
3010             if len(changes) == 0:
3011                 if not self.silent:
3012                     print "No changes to import!"
3013             else:
3014                 if not self.silent and not self.detectBranches:
3015                     print "Import destination: %s" % self.branch
3016
3017                 self.updatedBranches = set()
3018
3019                 if not self.detectBranches:
3020                     if args:
3021                         # start a new branch
3022                         self.initialParent = ""
3023                     else:
3024                         # build on a previous revision
3025                         self.initialParent = parseRevision(self.branch)
3026
3027                 self.importChanges(changes)
3028
3029                 if not self.silent:
3030                     print ""
3031                     if len(self.updatedBranches) > 0:
3032                         sys.stdout.write("Updated branches: ")
3033                         for b in self.updatedBranches:
3034                             sys.stdout.write("%s " % b)
3035                         sys.stdout.write("\n")
3036
3037         if gitConfigBool("git-p4.importLabels"):
3038             self.importLabels = True
3039
3040         if self.importLabels:
3041             p4Labels = getP4Labels(self.depotPaths)
3042             gitTags = getGitTags()
3043
3044             missingP4Labels = p4Labels - gitTags
3045             self.importP4Labels(self.gitStream, missingP4Labels)
3046
3047         self.gitStream.close()
3048         if self.importProcess.wait() != 0:
3049             die("fast-import failed: %s" % self.gitError.read())
3050         self.gitOutput.close()
3051         self.gitError.close()
3052
3053         # Cleanup temporary branches created during import
3054         if self.tempBranches != []:
3055             for branch in self.tempBranches:
3056                 read_pipe("git update-ref -d %s" % branch)
3057             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3058
3059         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3060         # a convenient shortcut refname "p4".
3061         if self.importIntoRemotes:
3062             head_ref = self.refPrefix + "HEAD"
3063             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3064                 system(["git", "symbolic-ref", head_ref, self.branch])
3065
3066         return True
3067
3068 class P4Rebase(Command):
3069     def __init__(self):
3070         Command.__init__(self)
3071         self.options = [
3072                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3073         ]
3074         self.importLabels = False
3075         self.description = ("Fetches the latest revision from perforce and "
3076                             + "rebases the current work (branch) against it")
3077
3078     def run(self, args):
3079         sync = P4Sync()
3080         sync.importLabels = self.importLabels
3081         sync.run([])
3082
3083         return self.rebase()
3084
3085     def rebase(self):
3086         if os.system("git update-index --refresh") != 0:
3087             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3088         if len(read_pipe("git diff-index HEAD --")) > 0:
3089             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3090
3091         [upstream, settings] = findUpstreamBranchPoint()
3092         if len(upstream) == 0:
3093             die("Cannot find upstream branchpoint for rebase")
3094
3095         # the branchpoint may be p4/foo~3, so strip off the parent
3096         upstream = re.sub("~[0-9]+$", "", upstream)
3097
3098         print "Rebasing the current branch onto %s" % upstream
3099         oldHead = read_pipe("git rev-parse HEAD").strip()
3100         system("git rebase %s" % upstream)
3101         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3102         return True
3103
3104 class P4Clone(P4Sync):
3105     def __init__(self):
3106         P4Sync.__init__(self)
3107         self.description = "Creates a new git repository and imports from Perforce into it"
3108         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3109         self.options += [
3110             optparse.make_option("--destination", dest="cloneDestination",
3111                                  action='store', default=None,
3112                                  help="where to leave result of the clone"),
3113             optparse.make_option("--bare", dest="cloneBare",
3114                                  action="store_true", default=False),
3115         ]
3116         self.cloneDestination = None
3117         self.needsGit = False
3118         self.cloneBare = False
3119
3120     def defaultDestination(self, args):
3121         ## TODO: use common prefix of args?
3122         depotPath = args[0]
3123         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3124         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3125         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3126         depotDir = re.sub(r"/$", "", depotDir)
3127         return os.path.split(depotDir)[1]
3128
3129     def run(self, args):
3130         if len(args) < 1:
3131             return False
3132
3133         if self.keepRepoPath and not self.cloneDestination:
3134             sys.stderr.write("Must specify destination for --keep-path\n")
3135             sys.exit(1)
3136
3137         depotPaths = args
3138
3139         if not self.cloneDestination and len(depotPaths) > 1:
3140             self.cloneDestination = depotPaths[-1]
3141             depotPaths = depotPaths[:-1]
3142
3143         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3144         for p in depotPaths:
3145             if not p.startswith("//"):
3146                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3147                 return False
3148
3149         if not self.cloneDestination:
3150             self.cloneDestination = self.defaultDestination(args)
3151
3152         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3153
3154         if not os.path.exists(self.cloneDestination):
3155             os.makedirs(self.cloneDestination)
3156         chdir(self.cloneDestination)
3157
3158         init_cmd = [ "git", "init" ]
3159         if self.cloneBare:
3160             init_cmd.append("--bare")
3161         retcode = subprocess.call(init_cmd)
3162         if retcode:
3163             raise CalledProcessError(retcode, init_cmd)
3164
3165         if not P4Sync.run(self, depotPaths):
3166             return False
3167
3168         # create a master branch and check out a work tree
3169         if gitBranchExists(self.branch):
3170             system([ "git", "branch", "master", self.branch ])
3171             if not self.cloneBare:
3172                 system([ "git", "checkout", "-f" ])
3173         else:
3174             print 'Not checking out any branch, use ' \
3175                   '"git checkout -q -b master <branch>"'
3176
3177         # auto-set this variable if invoked with --use-client-spec
3178         if self.useClientSpec_from_options:
3179             system("git config --bool git-p4.useclientspec true")
3180
3181         return True
3182
3183 class P4Branches(Command):
3184     def __init__(self):
3185         Command.__init__(self)
3186         self.options = [ ]
3187         self.description = ("Shows the git branches that hold imports and their "
3188                             + "corresponding perforce depot paths")
3189         self.verbose = False
3190
3191     def run(self, args):
3192         if originP4BranchesExist():
3193             createOrUpdateBranchesFromOrigin()
3194
3195         cmdline = "git rev-parse --symbolic "
3196         cmdline += " --remotes"
3197
3198         for line in read_pipe_lines(cmdline):
3199             line = line.strip()
3200
3201             if not line.startswith('p4/') or line == "p4/HEAD":
3202                 continue
3203             branch = line
3204
3205             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3206             settings = extractSettingsGitLog(log)
3207
3208             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3209         return True
3210
3211 class HelpFormatter(optparse.IndentedHelpFormatter):
3212     def __init__(self):
3213         optparse.IndentedHelpFormatter.__init__(self)
3214
3215     def format_description(self, description):
3216         if description:
3217             return description + "\n"
3218         else:
3219             return ""
3220
3221 def printUsage(commands):
3222     print "usage: %s <command> [options]" % sys.argv[0]
3223     print ""
3224     print "valid commands: %s" % ", ".join(commands)
3225     print ""
3226     print "Try %s <command> --help for command specific help." % sys.argv[0]
3227     print ""
3228
3229 commands = {
3230     "debug" : P4Debug,
3231     "submit" : P4Submit,
3232     "commit" : P4Submit,
3233     "sync" : P4Sync,
3234     "rebase" : P4Rebase,
3235     "clone" : P4Clone,
3236     "rollback" : P4RollBack,
3237     "branches" : P4Branches
3238 }
3239
3240
3241 def main():
3242     if len(sys.argv[1:]) == 0:
3243         printUsage(commands.keys())
3244         sys.exit(2)
3245
3246     cmdName = sys.argv[1]
3247     try:
3248         klass = commands[cmdName]
3249         cmd = klass()
3250     except KeyError:
3251         print "unknown command %s" % cmdName
3252         print ""
3253         printUsage(commands.keys())
3254         sys.exit(2)
3255
3256     options = cmd.options
3257     cmd.gitdir = os.environ.get("GIT_DIR", None)
3258
3259     args = sys.argv[2:]
3260
3261     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3262     if cmd.needsGit:
3263         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3264
3265     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3266                                    options,
3267                                    description = cmd.description,
3268                                    formatter = HelpFormatter())
3269
3270     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3271     global verbose
3272     verbose = cmd.verbose
3273     if cmd.needsGit:
3274         if cmd.gitdir == None:
3275             cmd.gitdir = os.path.abspath(".git")
3276             if not isValidGitDir(cmd.gitdir):
3277                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3278                 if os.path.exists(cmd.gitdir):
3279                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3280                     if len(cdup) > 0:
3281                         chdir(cdup);
3282
3283         if not isValidGitDir(cmd.gitdir):
3284             if isValidGitDir(cmd.gitdir + "/.git"):
3285                 cmd.gitdir += "/.git"
3286             else:
3287                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3288
3289         os.environ["GIT_DIR"] = cmd.gitdir
3290
3291     if not cmd.run(args):
3292         parser.print_help()
3293         sys.exit(2)
3294
3295
3296 if __name__ == '__main__':
3297     main()