Merge branch 'sb/http-flaky-test-fix' into maint
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27     from subprocess import CalledProcessError
28 except ImportError:
29     # from python2.7:subprocess.py
30     # Exception classes used by this module.
31     class CalledProcessError(Exception):
32         """This exception is raised when a process run by check_call() returns
33         a non-zero exit status.  The exit status will be stored in the
34         returncode attribute."""
35         def __init__(self, returncode, cmd):
36             self.returncode = returncode
37             self.cmd = cmd
38         def __str__(self):
39             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 # Grab changes in blocks of this many revisions, unless otherwise requested
47 defaultBlockSize = 512
48
49 def p4_build_cmd(cmd):
50     """Build a suitable p4 command line.
51
52     This consolidates building and returning a p4 command line into one
53     location. It means that hooking into the environment, or other configuration
54     can be done more easily.
55     """
56     real_cmd = ["p4"]
57
58     user = gitConfig("git-p4.user")
59     if len(user) > 0:
60         real_cmd += ["-u",user]
61
62     password = gitConfig("git-p4.password")
63     if len(password) > 0:
64         real_cmd += ["-P", password]
65
66     port = gitConfig("git-p4.port")
67     if len(port) > 0:
68         real_cmd += ["-p", port]
69
70     host = gitConfig("git-p4.host")
71     if len(host) > 0:
72         real_cmd += ["-H", host]
73
74     client = gitConfig("git-p4.client")
75     if len(client) > 0:
76         real_cmd += ["-c", client]
77
78
79     if isinstance(cmd,basestring):
80         real_cmd = ' '.join(real_cmd) + ' ' + cmd
81     else:
82         real_cmd += cmd
83     return real_cmd
84
85 def chdir(path, is_client_path=False):
86     """Do chdir to the given path, and set the PWD environment
87        variable for use by P4.  It does not look at getcwd() output.
88        Since we're not using the shell, it is necessary to set the
89        PWD environment variable explicitly.
90
91        Normally, expand the path to force it to be absolute.  This
92        addresses the use of relative path names inside P4 settings,
93        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
94        as given; it looks for .p4config using PWD.
95
96        If is_client_path, the path was handed to us directly by p4,
97        and may be a symbolic link.  Do not call os.getcwd() in this
98        case, because it will cause p4 to think that PWD is not inside
99        the client path.
100        """
101
102     os.chdir(path)
103     if not is_client_path:
104         path = os.getcwd()
105     os.environ['PWD'] = path
106
107 def die(msg):
108     if verbose:
109         raise Exception(msg)
110     else:
111         sys.stderr.write(msg + "\n")
112         sys.exit(1)
113
114 def write_pipe(c, stdin):
115     if verbose:
116         sys.stderr.write('Writing pipe: %s\n' % str(c))
117
118     expand = isinstance(c,basestring)
119     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
120     pipe = p.stdin
121     val = pipe.write(stdin)
122     pipe.close()
123     if p.wait():
124         die('Command failed: %s' % str(c))
125
126     return val
127
128 def p4_write_pipe(c, stdin):
129     real_cmd = p4_build_cmd(c)
130     return write_pipe(real_cmd, stdin)
131
132 def read_pipe(c, ignore_error=False):
133     if verbose:
134         sys.stderr.write('Reading pipe: %s\n' % str(c))
135
136     expand = isinstance(c,basestring)
137     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
138     pipe = p.stdout
139     val = pipe.read()
140     if p.wait() and not ignore_error:
141         die('Command failed: %s' % str(c))
142
143     return val
144
145 def p4_read_pipe(c, ignore_error=False):
146     real_cmd = p4_build_cmd(c)
147     return read_pipe(real_cmd, ignore_error)
148
149 def read_pipe_lines(c):
150     if verbose:
151         sys.stderr.write('Reading pipe: %s\n' % str(c))
152
153     expand = isinstance(c, basestring)
154     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
155     pipe = p.stdout
156     val = pipe.readlines()
157     if pipe.close() or p.wait():
158         die('Command failed: %s' % str(c))
159
160     return val
161
162 def p4_read_pipe_lines(c):
163     """Specifically invoke p4 on the command supplied. """
164     real_cmd = p4_build_cmd(c)
165     return read_pipe_lines(real_cmd)
166
167 def p4_has_command(cmd):
168     """Ask p4 for help on this command.  If it returns an error, the
169        command does not exist in this version of p4."""
170     real_cmd = p4_build_cmd(["help", cmd])
171     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
172                                    stderr=subprocess.PIPE)
173     p.communicate()
174     return p.returncode == 0
175
176 def p4_has_move_command():
177     """See if the move command exists, that it supports -k, and that
178        it has not been administratively disabled.  The arguments
179        must be correct, but the filenames do not have to exist.  Use
180        ones with wildcards so even if they exist, it will fail."""
181
182     if not p4_has_command("move"):
183         return False
184     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
185     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
186     (out, err) = p.communicate()
187     # return code will be 1 in either case
188     if err.find("Invalid option") >= 0:
189         return False
190     if err.find("disabled") >= 0:
191         return False
192     # assume it failed because @... was invalid changelist
193     return True
194
195 def system(cmd):
196     expand = isinstance(cmd,basestring)
197     if verbose:
198         sys.stderr.write("executing %s\n" % str(cmd))
199     retcode = subprocess.call(cmd, shell=expand)
200     if retcode:
201         raise CalledProcessError(retcode, cmd)
202
203 def p4_system(cmd):
204     """Specifically invoke p4 as the system command. """
205     real_cmd = p4_build_cmd(cmd)
206     expand = isinstance(real_cmd, basestring)
207     retcode = subprocess.call(real_cmd, shell=expand)
208     if retcode:
209         raise CalledProcessError(retcode, real_cmd)
210
211 _p4_version_string = None
212 def p4_version_string():
213     """Read the version string, showing just the last line, which
214        hopefully is the interesting version bit.
215
216        $ p4 -V
217        Perforce - The Fast Software Configuration Management System.
218        Copyright 1995-2011 Perforce Software.  All rights reserved.
219        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
220     """
221     global _p4_version_string
222     if not _p4_version_string:
223         a = p4_read_pipe_lines(["-V"])
224         _p4_version_string = a[-1].rstrip()
225     return _p4_version_string
226
227 def p4_integrate(src, dest):
228     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
229
230 def p4_sync(f, *options):
231     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
232
233 def p4_add(f):
234     # forcibly add file names with wildcards
235     if wildcard_present(f):
236         p4_system(["add", "-f", f])
237     else:
238         p4_system(["add", f])
239
240 def p4_delete(f):
241     p4_system(["delete", wildcard_encode(f)])
242
243 def p4_edit(f):
244     p4_system(["edit", wildcard_encode(f)])
245
246 def p4_revert(f):
247     p4_system(["revert", wildcard_encode(f)])
248
249 def p4_reopen(type, f):
250     p4_system(["reopen", "-t", type, wildcard_encode(f)])
251
252 def p4_move(src, dest):
253     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
254
255 def p4_last_change():
256     results = p4CmdList(["changes", "-m", "1"])
257     return int(results[0]['change'])
258
259 def p4_describe(change):
260     """Make sure it returns a valid result by checking for
261        the presence of field "time".  Return a dict of the
262        results."""
263
264     ds = p4CmdList(["describe", "-s", str(change)])
265     if len(ds) != 1:
266         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
267
268     d = ds[0]
269
270     if "p4ExitCode" in d:
271         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
272                                                       str(d)))
273     if "code" in d:
274         if d["code"] == "error":
275             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
276
277     if "time" not in d:
278         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
279
280     return d
281
282 #
283 # Canonicalize the p4 type and return a tuple of the
284 # base type, plus any modifiers.  See "p4 help filetypes"
285 # for a list and explanation.
286 #
287 def split_p4_type(p4type):
288
289     p4_filetypes_historical = {
290         "ctempobj": "binary+Sw",
291         "ctext": "text+C",
292         "cxtext": "text+Cx",
293         "ktext": "text+k",
294         "kxtext": "text+kx",
295         "ltext": "text+F",
296         "tempobj": "binary+FSw",
297         "ubinary": "binary+F",
298         "uresource": "resource+F",
299         "uxbinary": "binary+Fx",
300         "xbinary": "binary+x",
301         "xltext": "text+Fx",
302         "xtempobj": "binary+Swx",
303         "xtext": "text+x",
304         "xunicode": "unicode+x",
305         "xutf16": "utf16+x",
306     }
307     if p4type in p4_filetypes_historical:
308         p4type = p4_filetypes_historical[p4type]
309     mods = ""
310     s = p4type.split("+")
311     base = s[0]
312     mods = ""
313     if len(s) > 1:
314         mods = s[1]
315     return (base, mods)
316
317 #
318 # return the raw p4 type of a file (text, text+ko, etc)
319 #
320 def p4_type(f):
321     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
322     return results[0]['headType']
323
324 #
325 # Given a type base and modifier, return a regexp matching
326 # the keywords that can be expanded in the file
327 #
328 def p4_keywords_regexp_for_type(base, type_mods):
329     if base in ("text", "unicode", "binary"):
330         kwords = None
331         if "ko" in type_mods:
332             kwords = 'Id|Header'
333         elif "k" in type_mods:
334             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
335         else:
336             return None
337         pattern = r"""
338             \$              # Starts with a dollar, followed by...
339             (%s)            # one of the keywords, followed by...
340             (:[^$\n]+)?     # possibly an old expansion, followed by...
341             \$              # another dollar
342             """ % kwords
343         return pattern
344     else:
345         return None
346
347 #
348 # Given a file, return a regexp matching the possible
349 # RCS keywords that will be expanded, or None for files
350 # with kw expansion turned off.
351 #
352 def p4_keywords_regexp_for_file(file):
353     if not os.path.exists(file):
354         return None
355     else:
356         (type_base, type_mods) = split_p4_type(p4_type(file))
357         return p4_keywords_regexp_for_type(type_base, type_mods)
358
359 def setP4ExecBit(file, mode):
360     # Reopens an already open file and changes the execute bit to match
361     # the execute bit setting in the passed in mode.
362
363     p4Type = "+x"
364
365     if not isModeExec(mode):
366         p4Type = getP4OpenedType(file)
367         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
368         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
369         if p4Type[-1] == "+":
370             p4Type = p4Type[0:-1]
371
372     p4_reopen(p4Type, file)
373
374 def getP4OpenedType(file):
375     # Returns the perforce file type for the given file.
376
377     result = p4_read_pipe(["opened", wildcard_encode(file)])
378     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
379     if match:
380         return match.group(1)
381     else:
382         die("Could not determine file type for %s (result: '%s')" % (file, result))
383
384 # Return the set of all p4 labels
385 def getP4Labels(depotPaths):
386     labels = set()
387     if isinstance(depotPaths,basestring):
388         depotPaths = [depotPaths]
389
390     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
391         label = l['label']
392         labels.add(label)
393
394     return labels
395
396 # Return the set of all git tags
397 def getGitTags():
398     gitTags = set()
399     for line in read_pipe_lines(["git", "tag"]):
400         tag = line.strip()
401         gitTags.add(tag)
402     return gitTags
403
404 def diffTreePattern():
405     # This is a simple generator for the diff tree regex pattern. This could be
406     # a class variable if this and parseDiffTreeEntry were a part of a class.
407     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
408     while True:
409         yield pattern
410
411 def parseDiffTreeEntry(entry):
412     """Parses a single diff tree entry into its component elements.
413
414     See git-diff-tree(1) manpage for details about the format of the diff
415     output. This method returns a dictionary with the following elements:
416
417     src_mode - The mode of the source file
418     dst_mode - The mode of the destination file
419     src_sha1 - The sha1 for the source file
420     dst_sha1 - The sha1 fr the destination file
421     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
422     status_score - The score for the status (applicable for 'C' and 'R'
423                    statuses). This is None if there is no score.
424     src - The path for the source file.
425     dst - The path for the destination file. This is only present for
426           copy or renames. If it is not present, this is None.
427
428     If the pattern is not matched, None is returned."""
429
430     match = diffTreePattern().next().match(entry)
431     if match:
432         return {
433             'src_mode': match.group(1),
434             'dst_mode': match.group(2),
435             'src_sha1': match.group(3),
436             'dst_sha1': match.group(4),
437             'status': match.group(5),
438             'status_score': match.group(6),
439             'src': match.group(7),
440             'dst': match.group(10)
441         }
442     return None
443
444 def isModeExec(mode):
445     # Returns True if the given git mode represents an executable file,
446     # otherwise False.
447     return mode[-3:] == "755"
448
449 def isModeExecChanged(src_mode, dst_mode):
450     return isModeExec(src_mode) != isModeExec(dst_mode)
451
452 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
453
454     if isinstance(cmd,basestring):
455         cmd = "-G " + cmd
456         expand = True
457     else:
458         cmd = ["-G"] + cmd
459         expand = False
460
461     cmd = p4_build_cmd(cmd)
462     if verbose:
463         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
464
465     # Use a temporary file to avoid deadlocks without
466     # subprocess.communicate(), which would put another copy
467     # of stdout into memory.
468     stdin_file = None
469     if stdin is not None:
470         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
471         if isinstance(stdin,basestring):
472             stdin_file.write(stdin)
473         else:
474             for i in stdin:
475                 stdin_file.write(i + '\n')
476         stdin_file.flush()
477         stdin_file.seek(0)
478
479     p4 = subprocess.Popen(cmd,
480                           shell=expand,
481                           stdin=stdin_file,
482                           stdout=subprocess.PIPE)
483
484     result = []
485     try:
486         while True:
487             entry = marshal.load(p4.stdout)
488             if cb is not None:
489                 cb(entry)
490             else:
491                 result.append(entry)
492     except EOFError:
493         pass
494     exitCode = p4.wait()
495     if exitCode != 0:
496         entry = {}
497         entry["p4ExitCode"] = exitCode
498         result.append(entry)
499
500     return result
501
502 def p4Cmd(cmd):
503     list = p4CmdList(cmd)
504     result = {}
505     for entry in list:
506         result.update(entry)
507     return result;
508
509 def p4Where(depotPath):
510     if not depotPath.endswith("/"):
511         depotPath += "/"
512     depotPathLong = depotPath + "..."
513     outputList = p4CmdList(["where", depotPathLong])
514     output = None
515     for entry in outputList:
516         if "depotFile" in entry:
517             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
518             # The base path always ends with "/...".
519             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
520                 output = entry
521                 break
522         elif "data" in entry:
523             data = entry.get("data")
524             space = data.find(" ")
525             if data[:space] == depotPath:
526                 output = entry
527                 break
528     if output == None:
529         return ""
530     if output["code"] == "error":
531         return ""
532     clientPath = ""
533     if "path" in output:
534         clientPath = output.get("path")
535     elif "data" in output:
536         data = output.get("data")
537         lastSpace = data.rfind(" ")
538         clientPath = data[lastSpace + 1:]
539
540     if clientPath.endswith("..."):
541         clientPath = clientPath[:-3]
542     return clientPath
543
544 def currentGitBranch():
545     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
546
547 def isValidGitDir(path):
548     if (os.path.exists(path + "/HEAD")
549         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
550         return True;
551     return False
552
553 def parseRevision(ref):
554     return read_pipe("git rev-parse %s" % ref).strip()
555
556 def branchExists(ref):
557     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
558                      ignore_error=True)
559     return len(rev) > 0
560
561 def extractLogMessageFromGitCommit(commit):
562     logMessage = ""
563
564     ## fixme: title is first line of commit, not 1st paragraph.
565     foundTitle = False
566     for log in read_pipe_lines("git cat-file commit %s" % commit):
567        if not foundTitle:
568            if len(log) == 1:
569                foundTitle = True
570            continue
571
572        logMessage += log
573     return logMessage
574
575 def extractSettingsGitLog(log):
576     values = {}
577     for line in log.split("\n"):
578         line = line.strip()
579         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
580         if not m:
581             continue
582
583         assignments = m.group(1).split (':')
584         for a in assignments:
585             vals = a.split ('=')
586             key = vals[0].strip()
587             val = ('='.join (vals[1:])).strip()
588             if val.endswith ('\"') and val.startswith('"'):
589                 val = val[1:-1]
590
591             values[key] = val
592
593     paths = values.get("depot-paths")
594     if not paths:
595         paths = values.get("depot-path")
596     if paths:
597         values['depot-paths'] = paths.split(',')
598     return values
599
600 def gitBranchExists(branch):
601     proc = subprocess.Popen(["git", "rev-parse", branch],
602                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
603     return proc.wait() == 0;
604
605 _gitConfig = {}
606
607 def gitConfig(key):
608     if not _gitConfig.has_key(key):
609         cmd = [ "git", "config", key ]
610         s = read_pipe(cmd, ignore_error=True)
611         _gitConfig[key] = s.strip()
612     return _gitConfig[key]
613
614 def gitConfigBool(key):
615     """Return a bool, using git config --bool.  It is True only if the
616        variable is set to true, and False if set to false or not present
617        in the config."""
618
619     if not _gitConfig.has_key(key):
620         cmd = [ "git", "config", "--bool", key ]
621         s = read_pipe(cmd, ignore_error=True)
622         v = s.strip()
623         _gitConfig[key] = v == "true"
624     return _gitConfig[key]
625
626 def gitConfigList(key):
627     if not _gitConfig.has_key(key):
628         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
629         _gitConfig[key] = s.strip().split(os.linesep)
630     return _gitConfig[key]
631
632 def p4BranchesInGit(branchesAreInRemotes=True):
633     """Find all the branches whose names start with "p4/", looking
634        in remotes or heads as specified by the argument.  Return
635        a dictionary of { branch: revision } for each one found.
636        The branch names are the short names, without any
637        "p4/" prefix."""
638
639     branches = {}
640
641     cmdline = "git rev-parse --symbolic "
642     if branchesAreInRemotes:
643         cmdline += "--remotes"
644     else:
645         cmdline += "--branches"
646
647     for line in read_pipe_lines(cmdline):
648         line = line.strip()
649
650         # only import to p4/
651         if not line.startswith('p4/'):
652             continue
653         # special symbolic ref to p4/master
654         if line == "p4/HEAD":
655             continue
656
657         # strip off p4/ prefix
658         branch = line[len("p4/"):]
659
660         branches[branch] = parseRevision(line)
661
662     return branches
663
664 def branch_exists(branch):
665     """Make sure that the given ref name really exists."""
666
667     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
668     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
669     out, _ = p.communicate()
670     if p.returncode:
671         return False
672     # expect exactly one line of output: the branch name
673     return out.rstrip() == branch
674
675 def findUpstreamBranchPoint(head = "HEAD"):
676     branches = p4BranchesInGit()
677     # map from depot-path to branch name
678     branchByDepotPath = {}
679     for branch in branches.keys():
680         tip = branches[branch]
681         log = extractLogMessageFromGitCommit(tip)
682         settings = extractSettingsGitLog(log)
683         if settings.has_key("depot-paths"):
684             paths = ",".join(settings["depot-paths"])
685             branchByDepotPath[paths] = "remotes/p4/" + branch
686
687     settings = None
688     parent = 0
689     while parent < 65535:
690         commit = head + "~%s" % parent
691         log = extractLogMessageFromGitCommit(commit)
692         settings = extractSettingsGitLog(log)
693         if settings.has_key("depot-paths"):
694             paths = ",".join(settings["depot-paths"])
695             if branchByDepotPath.has_key(paths):
696                 return [branchByDepotPath[paths], settings]
697
698         parent = parent + 1
699
700     return ["", settings]
701
702 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
703     if not silent:
704         print ("Creating/updating branch(es) in %s based on origin branch(es)"
705                % localRefPrefix)
706
707     originPrefix = "origin/p4/"
708
709     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
710         line = line.strip()
711         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
712             continue
713
714         headName = line[len(originPrefix):]
715         remoteHead = localRefPrefix + headName
716         originHead = line
717
718         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
719         if (not original.has_key('depot-paths')
720             or not original.has_key('change')):
721             continue
722
723         update = False
724         if not gitBranchExists(remoteHead):
725             if verbose:
726                 print "creating %s" % remoteHead
727             update = True
728         else:
729             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
730             if settings.has_key('change') > 0:
731                 if settings['depot-paths'] == original['depot-paths']:
732                     originP4Change = int(original['change'])
733                     p4Change = int(settings['change'])
734                     if originP4Change > p4Change:
735                         print ("%s (%s) is newer than %s (%s). "
736                                "Updating p4 branch from origin."
737                                % (originHead, originP4Change,
738                                   remoteHead, p4Change))
739                         update = True
740                 else:
741                     print ("Ignoring: %s was imported from %s while "
742                            "%s was imported from %s"
743                            % (originHead, ','.join(original['depot-paths']),
744                               remoteHead, ','.join(settings['depot-paths'])))
745
746         if update:
747             system("git update-ref %s %s" % (remoteHead, originHead))
748
749 def originP4BranchesExist():
750         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
751
752
753 def p4ParseNumericChangeRange(parts):
754     changeStart = int(parts[0][1:])
755     if parts[1] == '#head':
756         changeEnd = p4_last_change()
757     else:
758         changeEnd = int(parts[1])
759
760     return (changeStart, changeEnd)
761
762 def chooseBlockSize(blockSize):
763     if blockSize:
764         return blockSize
765     else:
766         return defaultBlockSize
767
768 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
769     assert depotPaths
770
771     # Parse the change range into start and end. Try to find integer
772     # revision ranges as these can be broken up into blocks to avoid
773     # hitting server-side limits (maxrows, maxscanresults). But if
774     # that doesn't work, fall back to using the raw revision specifier
775     # strings, without using block mode.
776
777     if changeRange is None or changeRange == '':
778         changeStart = 1
779         changeEnd = p4_last_change()
780         block_size = chooseBlockSize(requestedBlockSize)
781     else:
782         parts = changeRange.split(',')
783         assert len(parts) == 2
784         try:
785             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
786             block_size = chooseBlockSize(requestedBlockSize)
787         except:
788             changeStart = parts[0][1:]
789             changeEnd = parts[1]
790             if requestedBlockSize:
791                 die("cannot use --changes-block-size with non-numeric revisions")
792             block_size = None
793
794     # Accumulate change numbers in a dictionary to avoid duplicates
795     changes = {}
796
797     for p in depotPaths:
798         # Retrieve changes a block at a time, to prevent running
799         # into a MaxResults/MaxScanRows error from the server.
800
801         while True:
802             cmd = ['changes']
803
804             if block_size:
805                 end = min(changeEnd, changeStart + block_size)
806                 revisionRange = "%d,%d" % (changeStart, end)
807             else:
808                 revisionRange = "%s,%s" % (changeStart, changeEnd)
809
810             cmd += ["%s...@%s" % (p, revisionRange)]
811
812             for line in p4_read_pipe_lines(cmd):
813                 changeNum = int(line.split(" ")[1])
814                 changes[changeNum] = True
815
816             if not block_size:
817                 break
818
819             if end >= changeEnd:
820                 break
821
822             changeStart = end + 1
823
824     changelist = changes.keys()
825     changelist.sort()
826     return changelist
827
828 def p4PathStartsWith(path, prefix):
829     # This method tries to remedy a potential mixed-case issue:
830     #
831     # If UserA adds  //depot/DirA/file1
832     # and UserB adds //depot/dira/file2
833     #
834     # we may or may not have a problem. If you have core.ignorecase=true,
835     # we treat DirA and dira as the same directory
836     if gitConfigBool("core.ignorecase"):
837         return path.lower().startswith(prefix.lower())
838     return path.startswith(prefix)
839
840 def getClientSpec():
841     """Look at the p4 client spec, create a View() object that contains
842        all the mappings, and return it."""
843
844     specList = p4CmdList("client -o")
845     if len(specList) != 1:
846         die('Output from "client -o" is %d lines, expecting 1' %
847             len(specList))
848
849     # dictionary of all client parameters
850     entry = specList[0]
851
852     # the //client/ name
853     client_name = entry["Client"]
854
855     # just the keys that start with "View"
856     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
857
858     # hold this new View
859     view = View(client_name)
860
861     # append the lines, in order, to the view
862     for view_num in range(len(view_keys)):
863         k = "View%d" % view_num
864         if k not in view_keys:
865             die("Expected view key %s missing" % k)
866         view.append(entry[k])
867
868     return view
869
870 def getClientRoot():
871     """Grab the client directory."""
872
873     output = p4CmdList("client -o")
874     if len(output) != 1:
875         die('Output from "client -o" is %d lines, expecting 1' % len(output))
876
877     entry = output[0]
878     if "Root" not in entry:
879         die('Client has no "Root"')
880
881     return entry["Root"]
882
883 #
884 # P4 wildcards are not allowed in filenames.  P4 complains
885 # if you simply add them, but you can force it with "-f", in
886 # which case it translates them into %xx encoding internally.
887 #
888 def wildcard_decode(path):
889     # Search for and fix just these four characters.  Do % last so
890     # that fixing it does not inadvertently create new %-escapes.
891     # Cannot have * in a filename in windows; untested as to
892     # what p4 would do in such a case.
893     if not platform.system() == "Windows":
894         path = path.replace("%2A", "*")
895     path = path.replace("%23", "#") \
896                .replace("%40", "@") \
897                .replace("%25", "%")
898     return path
899
900 def wildcard_encode(path):
901     # do % first to avoid double-encoding the %s introduced here
902     path = path.replace("%", "%25") \
903                .replace("*", "%2A") \
904                .replace("#", "%23") \
905                .replace("@", "%40")
906     return path
907
908 def wildcard_present(path):
909     m = re.search("[*#@%]", path)
910     return m is not None
911
912 class Command:
913     def __init__(self):
914         self.usage = "usage: %prog [options]"
915         self.needsGit = True
916         self.verbose = False
917
918 class P4UserMap:
919     def __init__(self):
920         self.userMapFromPerforceServer = False
921         self.myP4UserId = None
922
923     def p4UserId(self):
924         if self.myP4UserId:
925             return self.myP4UserId
926
927         results = p4CmdList("user -o")
928         for r in results:
929             if r.has_key('User'):
930                 self.myP4UserId = r['User']
931                 return r['User']
932         die("Could not find your p4 user id")
933
934     def p4UserIsMe(self, p4User):
935         # return True if the given p4 user is actually me
936         me = self.p4UserId()
937         if not p4User or p4User != me:
938             return False
939         else:
940             return True
941
942     def getUserCacheFilename(self):
943         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
944         return home + "/.gitp4-usercache.txt"
945
946     def getUserMapFromPerforceServer(self):
947         if self.userMapFromPerforceServer:
948             return
949         self.users = {}
950         self.emails = {}
951
952         for output in p4CmdList("users"):
953             if not output.has_key("User"):
954                 continue
955             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
956             self.emails[output["Email"]] = output["User"]
957
958
959         s = ''
960         for (key, val) in self.users.items():
961             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
962
963         open(self.getUserCacheFilename(), "wb").write(s)
964         self.userMapFromPerforceServer = True
965
966     def loadUserMapFromCache(self):
967         self.users = {}
968         self.userMapFromPerforceServer = False
969         try:
970             cache = open(self.getUserCacheFilename(), "rb")
971             lines = cache.readlines()
972             cache.close()
973             for line in lines:
974                 entry = line.strip().split("\t")
975                 self.users[entry[0]] = entry[1]
976         except IOError:
977             self.getUserMapFromPerforceServer()
978
979 class P4Debug(Command):
980     def __init__(self):
981         Command.__init__(self)
982         self.options = []
983         self.description = "A tool to debug the output of p4 -G."
984         self.needsGit = False
985
986     def run(self, args):
987         j = 0
988         for output in p4CmdList(args):
989             print 'Element: %d' % j
990             j += 1
991             print output
992         return True
993
994 class P4RollBack(Command):
995     def __init__(self):
996         Command.__init__(self)
997         self.options = [
998             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
999         ]
1000         self.description = "A tool to debug the multi-branch import. Don't use :)"
1001         self.rollbackLocalBranches = False
1002
1003     def run(self, args):
1004         if len(args) != 1:
1005             return False
1006         maxChange = int(args[0])
1007
1008         if "p4ExitCode" in p4Cmd("changes -m 1"):
1009             die("Problems executing p4");
1010
1011         if self.rollbackLocalBranches:
1012             refPrefix = "refs/heads/"
1013             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1014         else:
1015             refPrefix = "refs/remotes/"
1016             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1017
1018         for line in lines:
1019             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1020                 line = line.strip()
1021                 ref = refPrefix + line
1022                 log = extractLogMessageFromGitCommit(ref)
1023                 settings = extractSettingsGitLog(log)
1024
1025                 depotPaths = settings['depot-paths']
1026                 change = settings['change']
1027
1028                 changed = False
1029
1030                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1031                                                            for p in depotPaths]))) == 0:
1032                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1033                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1034                     continue
1035
1036                 while change and int(change) > maxChange:
1037                     changed = True
1038                     if self.verbose:
1039                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1040                     system("git update-ref %s \"%s^\"" % (ref, ref))
1041                     log = extractLogMessageFromGitCommit(ref)
1042                     settings =  extractSettingsGitLog(log)
1043
1044
1045                     depotPaths = settings['depot-paths']
1046                     change = settings['change']
1047
1048                 if changed:
1049                     print "%s rewound to %s" % (ref, change)
1050
1051         return True
1052
1053 class P4Submit(Command, P4UserMap):
1054
1055     conflict_behavior_choices = ("ask", "skip", "quit")
1056
1057     def __init__(self):
1058         Command.__init__(self)
1059         P4UserMap.__init__(self)
1060         self.options = [
1061                 optparse.make_option("--origin", dest="origin"),
1062                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1063                 # preserve the user, requires relevant p4 permissions
1064                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1065                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1066                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1067                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1068                 optparse.make_option("--conflict", dest="conflict_behavior",
1069                                      choices=self.conflict_behavior_choices),
1070                 optparse.make_option("--branch", dest="branch"),
1071         ]
1072         self.description = "Submit changes from git to the perforce depot."
1073         self.usage += " [name of git branch to submit into perforce depot]"
1074         self.origin = ""
1075         self.detectRenames = False
1076         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1077         self.dry_run = False
1078         self.prepare_p4_only = False
1079         self.conflict_behavior = None
1080         self.isWindows = (platform.system() == "Windows")
1081         self.exportLabels = False
1082         self.p4HasMoveCommand = p4_has_move_command()
1083         self.branch = None
1084
1085     def check(self):
1086         if len(p4CmdList("opened ...")) > 0:
1087             die("You have files opened with perforce! Close them before starting the sync.")
1088
1089     def separate_jobs_from_description(self, message):
1090         """Extract and return a possible Jobs field in the commit
1091            message.  It goes into a separate section in the p4 change
1092            specification.
1093
1094            A jobs line starts with "Jobs:" and looks like a new field
1095            in a form.  Values are white-space separated on the same
1096            line or on following lines that start with a tab.
1097
1098            This does not parse and extract the full git commit message
1099            like a p4 form.  It just sees the Jobs: line as a marker
1100            to pass everything from then on directly into the p4 form,
1101            but outside the description section.
1102
1103            Return a tuple (stripped log message, jobs string)."""
1104
1105         m = re.search(r'^Jobs:', message, re.MULTILINE)
1106         if m is None:
1107             return (message, None)
1108
1109         jobtext = message[m.start():]
1110         stripped_message = message[:m.start()].rstrip()
1111         return (stripped_message, jobtext)
1112
1113     def prepareLogMessage(self, template, message, jobs):
1114         """Edits the template returned from "p4 change -o" to insert
1115            the message in the Description field, and the jobs text in
1116            the Jobs field."""
1117         result = ""
1118
1119         inDescriptionSection = False
1120
1121         for line in template.split("\n"):
1122             if line.startswith("#"):
1123                 result += line + "\n"
1124                 continue
1125
1126             if inDescriptionSection:
1127                 if line.startswith("Files:") or line.startswith("Jobs:"):
1128                     inDescriptionSection = False
1129                     # insert Jobs section
1130                     if jobs:
1131                         result += jobs + "\n"
1132                 else:
1133                     continue
1134             else:
1135                 if line.startswith("Description:"):
1136                     inDescriptionSection = True
1137                     line += "\n"
1138                     for messageLine in message.split("\n"):
1139                         line += "\t" + messageLine + "\n"
1140
1141             result += line + "\n"
1142
1143         return result
1144
1145     def patchRCSKeywords(self, file, pattern):
1146         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1147         (handle, outFileName) = tempfile.mkstemp(dir='.')
1148         try:
1149             outFile = os.fdopen(handle, "w+")
1150             inFile = open(file, "r")
1151             regexp = re.compile(pattern, re.VERBOSE)
1152             for line in inFile.readlines():
1153                 line = regexp.sub(r'$\1$', line)
1154                 outFile.write(line)
1155             inFile.close()
1156             outFile.close()
1157             # Forcibly overwrite the original file
1158             os.unlink(file)
1159             shutil.move(outFileName, file)
1160         except:
1161             # cleanup our temporary file
1162             os.unlink(outFileName)
1163             print "Failed to strip RCS keywords in %s" % file
1164             raise
1165
1166         print "Patched up RCS keywords in %s" % file
1167
1168     def p4UserForCommit(self,id):
1169         # Return the tuple (perforce user,git email) for a given git commit id
1170         self.getUserMapFromPerforceServer()
1171         gitEmail = read_pipe(["git", "log", "--max-count=1",
1172                               "--format=%ae", id])
1173         gitEmail = gitEmail.strip()
1174         if not self.emails.has_key(gitEmail):
1175             return (None,gitEmail)
1176         else:
1177             return (self.emails[gitEmail],gitEmail)
1178
1179     def checkValidP4Users(self,commits):
1180         # check if any git authors cannot be mapped to p4 users
1181         for id in commits:
1182             (user,email) = self.p4UserForCommit(id)
1183             if not user:
1184                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1185                 if gitConfigBool("git-p4.allowMissingP4Users"):
1186                     print "%s" % msg
1187                 else:
1188                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1189
1190     def lastP4Changelist(self):
1191         # Get back the last changelist number submitted in this client spec. This
1192         # then gets used to patch up the username in the change. If the same
1193         # client spec is being used by multiple processes then this might go
1194         # wrong.
1195         results = p4CmdList("client -o")        # find the current client
1196         client = None
1197         for r in results:
1198             if r.has_key('Client'):
1199                 client = r['Client']
1200                 break
1201         if not client:
1202             die("could not get client spec")
1203         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1204         for r in results:
1205             if r.has_key('change'):
1206                 return r['change']
1207         die("Could not get changelist number for last submit - cannot patch up user details")
1208
1209     def modifyChangelistUser(self, changelist, newUser):
1210         # fixup the user field of a changelist after it has been submitted.
1211         changes = p4CmdList("change -o %s" % changelist)
1212         if len(changes) != 1:
1213             die("Bad output from p4 change modifying %s to user %s" %
1214                 (changelist, newUser))
1215
1216         c = changes[0]
1217         if c['User'] == newUser: return   # nothing to do
1218         c['User'] = newUser
1219         input = marshal.dumps(c)
1220
1221         result = p4CmdList("change -f -i", stdin=input)
1222         for r in result:
1223             if r.has_key('code'):
1224                 if r['code'] == 'error':
1225                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1226             if r.has_key('data'):
1227                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1228                 return
1229         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1230
1231     def canChangeChangelists(self):
1232         # check to see if we have p4 admin or super-user permissions, either of
1233         # which are required to modify changelists.
1234         results = p4CmdList(["protects", self.depotPath])
1235         for r in results:
1236             if r.has_key('perm'):
1237                 if r['perm'] == 'admin':
1238                     return 1
1239                 if r['perm'] == 'super':
1240                     return 1
1241         return 0
1242
1243     def prepareSubmitTemplate(self):
1244         """Run "p4 change -o" to grab a change specification template.
1245            This does not use "p4 -G", as it is nice to keep the submission
1246            template in original order, since a human might edit it.
1247
1248            Remove lines in the Files section that show changes to files
1249            outside the depot path we're committing into."""
1250
1251         template = ""
1252         inFilesSection = False
1253         for line in p4_read_pipe_lines(['change', '-o']):
1254             if line.endswith("\r\n"):
1255                 line = line[:-2] + "\n"
1256             if inFilesSection:
1257                 if line.startswith("\t"):
1258                     # path starts and ends with a tab
1259                     path = line[1:]
1260                     lastTab = path.rfind("\t")
1261                     if lastTab != -1:
1262                         path = path[:lastTab]
1263                         if not p4PathStartsWith(path, self.depotPath):
1264                             continue
1265                 else:
1266                     inFilesSection = False
1267             else:
1268                 if line.startswith("Files:"):
1269                     inFilesSection = True
1270
1271             template += line
1272
1273         return template
1274
1275     def edit_template(self, template_file):
1276         """Invoke the editor to let the user change the submission
1277            message.  Return true if okay to continue with the submit."""
1278
1279         # if configured to skip the editing part, just submit
1280         if gitConfigBool("git-p4.skipSubmitEdit"):
1281             return True
1282
1283         # look at the modification time, to check later if the user saved
1284         # the file
1285         mtime = os.stat(template_file).st_mtime
1286
1287         # invoke the editor
1288         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1289             editor = os.environ.get("P4EDITOR")
1290         else:
1291             editor = read_pipe("git var GIT_EDITOR").strip()
1292         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1293
1294         # If the file was not saved, prompt to see if this patch should
1295         # be skipped.  But skip this verification step if configured so.
1296         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1297             return True
1298
1299         # modification time updated means user saved the file
1300         if os.stat(template_file).st_mtime > mtime:
1301             return True
1302
1303         while True:
1304             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1305             if response == 'y':
1306                 return True
1307             if response == 'n':
1308                 return False
1309
1310     def get_diff_description(self, editedFiles, filesToAdd):
1311         # diff
1312         if os.environ.has_key("P4DIFF"):
1313             del(os.environ["P4DIFF"])
1314         diff = ""
1315         for editedFile in editedFiles:
1316             diff += p4_read_pipe(['diff', '-du',
1317                                   wildcard_encode(editedFile)])
1318
1319         # new file diff
1320         newdiff = ""
1321         for newFile in filesToAdd:
1322             newdiff += "==== new file ====\n"
1323             newdiff += "--- /dev/null\n"
1324             newdiff += "+++ %s\n" % newFile
1325             f = open(newFile, "r")
1326             for line in f.readlines():
1327                 newdiff += "+" + line
1328             f.close()
1329
1330         return (diff + newdiff).replace('\r\n', '\n')
1331
1332     def applyCommit(self, id):
1333         """Apply one commit, return True if it succeeded."""
1334
1335         print "Applying", read_pipe(["git", "show", "-s",
1336                                      "--format=format:%h %s", id])
1337
1338         (p4User, gitEmail) = self.p4UserForCommit(id)
1339
1340         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1341         filesToAdd = set()
1342         filesToDelete = set()
1343         editedFiles = set()
1344         pureRenameCopy = set()
1345         filesToChangeExecBit = {}
1346
1347         for line in diff:
1348             diff = parseDiffTreeEntry(line)
1349             modifier = diff['status']
1350             path = diff['src']
1351             if modifier == "M":
1352                 p4_edit(path)
1353                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1354                     filesToChangeExecBit[path] = diff['dst_mode']
1355                 editedFiles.add(path)
1356             elif modifier == "A":
1357                 filesToAdd.add(path)
1358                 filesToChangeExecBit[path] = diff['dst_mode']
1359                 if path in filesToDelete:
1360                     filesToDelete.remove(path)
1361             elif modifier == "D":
1362                 filesToDelete.add(path)
1363                 if path in filesToAdd:
1364                     filesToAdd.remove(path)
1365             elif modifier == "C":
1366                 src, dest = diff['src'], diff['dst']
1367                 p4_integrate(src, dest)
1368                 pureRenameCopy.add(dest)
1369                 if diff['src_sha1'] != diff['dst_sha1']:
1370                     p4_edit(dest)
1371                     pureRenameCopy.discard(dest)
1372                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1373                     p4_edit(dest)
1374                     pureRenameCopy.discard(dest)
1375                     filesToChangeExecBit[dest] = diff['dst_mode']
1376                 if self.isWindows:
1377                     # turn off read-only attribute
1378                     os.chmod(dest, stat.S_IWRITE)
1379                 os.unlink(dest)
1380                 editedFiles.add(dest)
1381             elif modifier == "R":
1382                 src, dest = diff['src'], diff['dst']
1383                 if self.p4HasMoveCommand:
1384                     p4_edit(src)        # src must be open before move
1385                     p4_move(src, dest)  # opens for (move/delete, move/add)
1386                 else:
1387                     p4_integrate(src, dest)
1388                     if diff['src_sha1'] != diff['dst_sha1']:
1389                         p4_edit(dest)
1390                     else:
1391                         pureRenameCopy.add(dest)
1392                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1393                     if not self.p4HasMoveCommand:
1394                         p4_edit(dest)   # with move: already open, writable
1395                     filesToChangeExecBit[dest] = diff['dst_mode']
1396                 if not self.p4HasMoveCommand:
1397                     if self.isWindows:
1398                         os.chmod(dest, stat.S_IWRITE)
1399                     os.unlink(dest)
1400                     filesToDelete.add(src)
1401                 editedFiles.add(dest)
1402             else:
1403                 die("unknown modifier %s for %s" % (modifier, path))
1404
1405         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1406         patchcmd = diffcmd + " | git apply "
1407         tryPatchCmd = patchcmd + "--check -"
1408         applyPatchCmd = patchcmd + "--check --apply -"
1409         patch_succeeded = True
1410
1411         if os.system(tryPatchCmd) != 0:
1412             fixed_rcs_keywords = False
1413             patch_succeeded = False
1414             print "Unfortunately applying the change failed!"
1415
1416             # Patch failed, maybe it's just RCS keyword woes. Look through
1417             # the patch to see if that's possible.
1418             if gitConfigBool("git-p4.attemptRCSCleanup"):
1419                 file = None
1420                 pattern = None
1421                 kwfiles = {}
1422                 for file in editedFiles | filesToDelete:
1423                     # did this file's delta contain RCS keywords?
1424                     pattern = p4_keywords_regexp_for_file(file)
1425
1426                     if pattern:
1427                         # this file is a possibility...look for RCS keywords.
1428                         regexp = re.compile(pattern, re.VERBOSE)
1429                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1430                             if regexp.search(line):
1431                                 if verbose:
1432                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1433                                 kwfiles[file] = pattern
1434                                 break
1435
1436                 for file in kwfiles:
1437                     if verbose:
1438                         print "zapping %s with %s" % (line,pattern)
1439                     # File is being deleted, so not open in p4.  Must
1440                     # disable the read-only bit on windows.
1441                     if self.isWindows and file not in editedFiles:
1442                         os.chmod(file, stat.S_IWRITE)
1443                     self.patchRCSKeywords(file, kwfiles[file])
1444                     fixed_rcs_keywords = True
1445
1446             if fixed_rcs_keywords:
1447                 print "Retrying the patch with RCS keywords cleaned up"
1448                 if os.system(tryPatchCmd) == 0:
1449                     patch_succeeded = True
1450
1451         if not patch_succeeded:
1452             for f in editedFiles:
1453                 p4_revert(f)
1454             return False
1455
1456         #
1457         # Apply the patch for real, and do add/delete/+x handling.
1458         #
1459         system(applyPatchCmd)
1460
1461         for f in filesToAdd:
1462             p4_add(f)
1463         for f in filesToDelete:
1464             p4_revert(f)
1465             p4_delete(f)
1466
1467         # Set/clear executable bits
1468         for f in filesToChangeExecBit.keys():
1469             mode = filesToChangeExecBit[f]
1470             setP4ExecBit(f, mode)
1471
1472         #
1473         # Build p4 change description, starting with the contents
1474         # of the git commit message.
1475         #
1476         logMessage = extractLogMessageFromGitCommit(id)
1477         logMessage = logMessage.strip()
1478         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1479
1480         template = self.prepareSubmitTemplate()
1481         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1482
1483         if self.preserveUser:
1484            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1485
1486         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1487             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1488             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1489             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1490
1491         separatorLine = "######## everything below this line is just the diff #######\n"
1492         if not self.prepare_p4_only:
1493             submitTemplate += separatorLine
1494             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1495
1496         (handle, fileName) = tempfile.mkstemp()
1497         tmpFile = os.fdopen(handle, "w+b")
1498         if self.isWindows:
1499             submitTemplate = submitTemplate.replace("\n", "\r\n")
1500         tmpFile.write(submitTemplate)
1501         tmpFile.close()
1502
1503         if self.prepare_p4_only:
1504             #
1505             # Leave the p4 tree prepared, and the submit template around
1506             # and let the user decide what to do next
1507             #
1508             print
1509             print "P4 workspace prepared for submission."
1510             print "To submit or revert, go to client workspace"
1511             print "  " + self.clientPath
1512             print
1513             print "To submit, use \"p4 submit\" to write a new description,"
1514             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1515                   " \"git p4\"." % fileName
1516             print "You can delete the file \"%s\" when finished." % fileName
1517
1518             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1519                 print "To preserve change ownership by user %s, you must\n" \
1520                       "do \"p4 change -f <change>\" after submitting and\n" \
1521                       "edit the User field."
1522             if pureRenameCopy:
1523                 print "After submitting, renamed files must be re-synced."
1524                 print "Invoke \"p4 sync -f\" on each of these files:"
1525                 for f in pureRenameCopy:
1526                     print "  " + f
1527
1528             print
1529             print "To revert the changes, use \"p4 revert ...\", and delete"
1530             print "the submit template file \"%s\"" % fileName
1531             if filesToAdd:
1532                 print "Since the commit adds new files, they must be deleted:"
1533                 for f in filesToAdd:
1534                     print "  " + f
1535             print
1536             return True
1537
1538         #
1539         # Let the user edit the change description, then submit it.
1540         #
1541         if self.edit_template(fileName):
1542             # read the edited message and submit
1543             ret = True
1544             tmpFile = open(fileName, "rb")
1545             message = tmpFile.read()
1546             tmpFile.close()
1547             if self.isWindows:
1548                 message = message.replace("\r\n", "\n")
1549             submitTemplate = message[:message.index(separatorLine)]
1550             p4_write_pipe(['submit', '-i'], submitTemplate)
1551
1552             if self.preserveUser:
1553                 if p4User:
1554                     # Get last changelist number. Cannot easily get it from
1555                     # the submit command output as the output is
1556                     # unmarshalled.
1557                     changelist = self.lastP4Changelist()
1558                     self.modifyChangelistUser(changelist, p4User)
1559
1560             # The rename/copy happened by applying a patch that created a
1561             # new file.  This leaves it writable, which confuses p4.
1562             for f in pureRenameCopy:
1563                 p4_sync(f, "-f")
1564
1565         else:
1566             # skip this patch
1567             ret = False
1568             print "Submission cancelled, undoing p4 changes."
1569             for f in editedFiles:
1570                 p4_revert(f)
1571             for f in filesToAdd:
1572                 p4_revert(f)
1573                 os.remove(f)
1574             for f in filesToDelete:
1575                 p4_revert(f)
1576
1577         os.remove(fileName)
1578         return ret
1579
1580     # Export git tags as p4 labels. Create a p4 label and then tag
1581     # with that.
1582     def exportGitTags(self, gitTags):
1583         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1584         if len(validLabelRegexp) == 0:
1585             validLabelRegexp = defaultLabelRegexp
1586         m = re.compile(validLabelRegexp)
1587
1588         for name in gitTags:
1589
1590             if not m.match(name):
1591                 if verbose:
1592                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1593                 continue
1594
1595             # Get the p4 commit this corresponds to
1596             logMessage = extractLogMessageFromGitCommit(name)
1597             values = extractSettingsGitLog(logMessage)
1598
1599             if not values.has_key('change'):
1600                 # a tag pointing to something not sent to p4; ignore
1601                 if verbose:
1602                     print "git tag %s does not give a p4 commit" % name
1603                 continue
1604             else:
1605                 changelist = values['change']
1606
1607             # Get the tag details.
1608             inHeader = True
1609             isAnnotated = False
1610             body = []
1611             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1612                 l = l.strip()
1613                 if inHeader:
1614                     if re.match(r'tag\s+', l):
1615                         isAnnotated = True
1616                     elif re.match(r'\s*$', l):
1617                         inHeader = False
1618                         continue
1619                 else:
1620                     body.append(l)
1621
1622             if not isAnnotated:
1623                 body = ["lightweight tag imported by git p4\n"]
1624
1625             # Create the label - use the same view as the client spec we are using
1626             clientSpec = getClientSpec()
1627
1628             labelTemplate  = "Label: %s\n" % name
1629             labelTemplate += "Description:\n"
1630             for b in body:
1631                 labelTemplate += "\t" + b + "\n"
1632             labelTemplate += "View:\n"
1633             for depot_side in clientSpec.mappings:
1634                 labelTemplate += "\t%s\n" % depot_side
1635
1636             if self.dry_run:
1637                 print "Would create p4 label %s for tag" % name
1638             elif self.prepare_p4_only:
1639                 print "Not creating p4 label %s for tag due to option" \
1640                       " --prepare-p4-only" % name
1641             else:
1642                 p4_write_pipe(["label", "-i"], labelTemplate)
1643
1644                 # Use the label
1645                 p4_system(["tag", "-l", name] +
1646                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1647
1648                 if verbose:
1649                     print "created p4 label for tag %s" % name
1650
1651     def run(self, args):
1652         if len(args) == 0:
1653             self.master = currentGitBranch()
1654             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1655                 die("Detecting current git branch failed!")
1656         elif len(args) == 1:
1657             self.master = args[0]
1658             if not branchExists(self.master):
1659                 die("Branch %s does not exist" % self.master)
1660         else:
1661             return False
1662
1663         allowSubmit = gitConfig("git-p4.allowSubmit")
1664         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1665             die("%s is not in git-p4.allowSubmit" % self.master)
1666
1667         [upstream, settings] = findUpstreamBranchPoint()
1668         self.depotPath = settings['depot-paths'][0]
1669         if len(self.origin) == 0:
1670             self.origin = upstream
1671
1672         if self.preserveUser:
1673             if not self.canChangeChangelists():
1674                 die("Cannot preserve user names without p4 super-user or admin permissions")
1675
1676         # if not set from the command line, try the config file
1677         if self.conflict_behavior is None:
1678             val = gitConfig("git-p4.conflict")
1679             if val:
1680                 if val not in self.conflict_behavior_choices:
1681                     die("Invalid value '%s' for config git-p4.conflict" % val)
1682             else:
1683                 val = "ask"
1684             self.conflict_behavior = val
1685
1686         if self.verbose:
1687             print "Origin branch is " + self.origin
1688
1689         if len(self.depotPath) == 0:
1690             print "Internal error: cannot locate perforce depot path from existing branches"
1691             sys.exit(128)
1692
1693         self.useClientSpec = False
1694         if gitConfigBool("git-p4.useclientspec"):
1695             self.useClientSpec = True
1696         if self.useClientSpec:
1697             self.clientSpecDirs = getClientSpec()
1698
1699         # Check for the existance of P4 branches
1700         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1701
1702         if self.useClientSpec and not branchesDetected:
1703             # all files are relative to the client spec
1704             self.clientPath = getClientRoot()
1705         else:
1706             self.clientPath = p4Where(self.depotPath)
1707
1708         if self.clientPath == "":
1709             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1710
1711         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1712         self.oldWorkingDirectory = os.getcwd()
1713
1714         # ensure the clientPath exists
1715         new_client_dir = False
1716         if not os.path.exists(self.clientPath):
1717             new_client_dir = True
1718             os.makedirs(self.clientPath)
1719
1720         chdir(self.clientPath, is_client_path=True)
1721         if self.dry_run:
1722             print "Would synchronize p4 checkout in %s" % self.clientPath
1723         else:
1724             print "Synchronizing p4 checkout..."
1725             if new_client_dir:
1726                 # old one was destroyed, and maybe nobody told p4
1727                 p4_sync("...", "-f")
1728             else:
1729                 p4_sync("...")
1730         self.check()
1731
1732         commits = []
1733         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1734             commits.append(line.strip())
1735         commits.reverse()
1736
1737         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1738             self.checkAuthorship = False
1739         else:
1740             self.checkAuthorship = True
1741
1742         if self.preserveUser:
1743             self.checkValidP4Users(commits)
1744
1745         #
1746         # Build up a set of options to be passed to diff when
1747         # submitting each commit to p4.
1748         #
1749         if self.detectRenames:
1750             # command-line -M arg
1751             self.diffOpts = "-M"
1752         else:
1753             # If not explicitly set check the config variable
1754             detectRenames = gitConfig("git-p4.detectRenames")
1755
1756             if detectRenames.lower() == "false" or detectRenames == "":
1757                 self.diffOpts = ""
1758             elif detectRenames.lower() == "true":
1759                 self.diffOpts = "-M"
1760             else:
1761                 self.diffOpts = "-M%s" % detectRenames
1762
1763         # no command-line arg for -C or --find-copies-harder, just
1764         # config variables
1765         detectCopies = gitConfig("git-p4.detectCopies")
1766         if detectCopies.lower() == "false" or detectCopies == "":
1767             pass
1768         elif detectCopies.lower() == "true":
1769             self.diffOpts += " -C"
1770         else:
1771             self.diffOpts += " -C%s" % detectCopies
1772
1773         if gitConfigBool("git-p4.detectCopiesHarder"):
1774             self.diffOpts += " --find-copies-harder"
1775
1776         #
1777         # Apply the commits, one at a time.  On failure, ask if should
1778         # continue to try the rest of the patches, or quit.
1779         #
1780         if self.dry_run:
1781             print "Would apply"
1782         applied = []
1783         last = len(commits) - 1
1784         for i, commit in enumerate(commits):
1785             if self.dry_run:
1786                 print " ", read_pipe(["git", "show", "-s",
1787                                       "--format=format:%h %s", commit])
1788                 ok = True
1789             else:
1790                 ok = self.applyCommit(commit)
1791             if ok:
1792                 applied.append(commit)
1793             else:
1794                 if self.prepare_p4_only and i < last:
1795                     print "Processing only the first commit due to option" \
1796                           " --prepare-p4-only"
1797                     break
1798                 if i < last:
1799                     quit = False
1800                     while True:
1801                         # prompt for what to do, or use the option/variable
1802                         if self.conflict_behavior == "ask":
1803                             print "What do you want to do?"
1804                             response = raw_input("[s]kip this commit but apply"
1805                                                  " the rest, or [q]uit? ")
1806                             if not response:
1807                                 continue
1808                         elif self.conflict_behavior == "skip":
1809                             response = "s"
1810                         elif self.conflict_behavior == "quit":
1811                             response = "q"
1812                         else:
1813                             die("Unknown conflict_behavior '%s'" %
1814                                 self.conflict_behavior)
1815
1816                         if response[0] == "s":
1817                             print "Skipping this commit, but applying the rest"
1818                             break
1819                         if response[0] == "q":
1820                             print "Quitting"
1821                             quit = True
1822                             break
1823                     if quit:
1824                         break
1825
1826         chdir(self.oldWorkingDirectory)
1827
1828         if self.dry_run:
1829             pass
1830         elif self.prepare_p4_only:
1831             pass
1832         elif len(commits) == len(applied):
1833             print "All commits applied!"
1834
1835             sync = P4Sync()
1836             if self.branch:
1837                 sync.branch = self.branch
1838             sync.run([])
1839
1840             rebase = P4Rebase()
1841             rebase.rebase()
1842
1843         else:
1844             if len(applied) == 0:
1845                 print "No commits applied."
1846             else:
1847                 print "Applied only the commits marked with '*':"
1848                 for c in commits:
1849                     if c in applied:
1850                         star = "*"
1851                     else:
1852                         star = " "
1853                     print star, read_pipe(["git", "show", "-s",
1854                                            "--format=format:%h %s",  c])
1855                 print "You will have to do 'git p4 sync' and rebase."
1856
1857         if gitConfigBool("git-p4.exportLabels"):
1858             self.exportLabels = True
1859
1860         if self.exportLabels:
1861             p4Labels = getP4Labels(self.depotPath)
1862             gitTags = getGitTags()
1863
1864             missingGitTags = gitTags - p4Labels
1865             self.exportGitTags(missingGitTags)
1866
1867         # exit with error unless everything applied perfectly
1868         if len(commits) != len(applied):
1869                 sys.exit(1)
1870
1871         return True
1872
1873 class View(object):
1874     """Represent a p4 view ("p4 help views"), and map files in a
1875        repo according to the view."""
1876
1877     def __init__(self, client_name):
1878         self.mappings = []
1879         self.client_prefix = "//%s/" % client_name
1880         # cache results of "p4 where" to lookup client file locations
1881         self.client_spec_path_cache = {}
1882
1883     def append(self, view_line):
1884         """Parse a view line, splitting it into depot and client
1885            sides.  Append to self.mappings, preserving order.  This
1886            is only needed for tag creation."""
1887
1888         # Split the view line into exactly two words.  P4 enforces
1889         # structure on these lines that simplifies this quite a bit.
1890         #
1891         # Either or both words may be double-quoted.
1892         # Single quotes do not matter.
1893         # Double-quote marks cannot occur inside the words.
1894         # A + or - prefix is also inside the quotes.
1895         # There are no quotes unless they contain a space.
1896         # The line is already white-space stripped.
1897         # The two words are separated by a single space.
1898         #
1899         if view_line[0] == '"':
1900             # First word is double quoted.  Find its end.
1901             close_quote_index = view_line.find('"', 1)
1902             if close_quote_index <= 0:
1903                 die("No first-word closing quote found: %s" % view_line)
1904             depot_side = view_line[1:close_quote_index]
1905             # skip closing quote and space
1906             rhs_index = close_quote_index + 1 + 1
1907         else:
1908             space_index = view_line.find(" ")
1909             if space_index <= 0:
1910                 die("No word-splitting space found: %s" % view_line)
1911             depot_side = view_line[0:space_index]
1912             rhs_index = space_index + 1
1913
1914         # prefix + means overlay on previous mapping
1915         if depot_side.startswith("+"):
1916             depot_side = depot_side[1:]
1917
1918         # prefix - means exclude this path, leave out of mappings
1919         exclude = False
1920         if depot_side.startswith("-"):
1921             exclude = True
1922             depot_side = depot_side[1:]
1923
1924         if not exclude:
1925             self.mappings.append(depot_side)
1926
1927     def convert_client_path(self, clientFile):
1928         # chop off //client/ part to make it relative
1929         if not clientFile.startswith(self.client_prefix):
1930             die("No prefix '%s' on clientFile '%s'" %
1931                 (self.client_prefix, clientFile))
1932         return clientFile[len(self.client_prefix):]
1933
1934     def update_client_spec_path_cache(self, files):
1935         """ Caching file paths by "p4 where" batch query """
1936
1937         # List depot file paths exclude that already cached
1938         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1939
1940         if len(fileArgs) == 0:
1941             return  # All files in cache
1942
1943         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1944         for res in where_result:
1945             if "code" in res and res["code"] == "error":
1946                 # assume error is "... file(s) not in client view"
1947                 continue
1948             if "clientFile" not in res:
1949                 die("No clientFile in 'p4 where' output")
1950             if "unmap" in res:
1951                 # it will list all of them, but only one not unmap-ped
1952                 continue
1953             if gitConfigBool("core.ignorecase"):
1954                 res['depotFile'] = res['depotFile'].lower()
1955             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1956
1957         # not found files or unmap files set to ""
1958         for depotFile in fileArgs:
1959             if gitConfigBool("core.ignorecase"):
1960                 depotFile = depotFile.lower()
1961             if depotFile not in self.client_spec_path_cache:
1962                 self.client_spec_path_cache[depotFile] = ""
1963
1964     def map_in_client(self, depot_path):
1965         """Return the relative location in the client where this
1966            depot file should live.  Returns "" if the file should
1967            not be mapped in the client."""
1968
1969         if gitConfigBool("core.ignorecase"):
1970             depot_path = depot_path.lower()
1971
1972         if depot_path in self.client_spec_path_cache:
1973             return self.client_spec_path_cache[depot_path]
1974
1975         die( "Error: %s is not found in client spec path" % depot_path )
1976         return ""
1977
1978 class P4Sync(Command, P4UserMap):
1979     delete_actions = ( "delete", "move/delete", "purge" )
1980
1981     def __init__(self):
1982         Command.__init__(self)
1983         P4UserMap.__init__(self)
1984         self.options = [
1985                 optparse.make_option("--branch", dest="branch"),
1986                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1987                 optparse.make_option("--changesfile", dest="changesFile"),
1988                 optparse.make_option("--silent", dest="silent", action="store_true"),
1989                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1990                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1991                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1992                                      help="Import into refs/heads/ , not refs/remotes"),
1993                 optparse.make_option("--max-changes", dest="maxChanges",
1994                                      help="Maximum number of changes to import"),
1995                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1996                                      help="Internal block size to use when iteratively calling p4 changes"),
1997                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1998                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1999                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2000                                      help="Only sync files that are included in the Perforce Client Spec"),
2001                 optparse.make_option("-/", dest="cloneExclude",
2002                                      action="append", type="string",
2003                                      help="exclude depot path"),
2004         ]
2005         self.description = """Imports from Perforce into a git repository.\n
2006     example:
2007     //depot/my/project/ -- to import the current head
2008     //depot/my/project/@all -- to import everything
2009     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2010
2011     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2012
2013         self.usage += " //depot/path[@revRange]"
2014         self.silent = False
2015         self.createdBranches = set()
2016         self.committedChanges = set()
2017         self.branch = ""
2018         self.detectBranches = False
2019         self.detectLabels = False
2020         self.importLabels = False
2021         self.changesFile = ""
2022         self.syncWithOrigin = True
2023         self.importIntoRemotes = True
2024         self.maxChanges = ""
2025         self.changes_block_size = None
2026         self.keepRepoPath = False
2027         self.depotPaths = None
2028         self.p4BranchesInGit = []
2029         self.cloneExclude = []
2030         self.useClientSpec = False
2031         self.useClientSpec_from_options = False
2032         self.clientSpecDirs = None
2033         self.tempBranches = []
2034         self.tempBranchLocation = "git-p4-tmp"
2035
2036         if gitConfig("git-p4.syncFromOrigin") == "false":
2037             self.syncWithOrigin = False
2038
2039     # This is required for the "append" cloneExclude action
2040     def ensure_value(self, attr, value):
2041         if not hasattr(self, attr) or getattr(self, attr) is None:
2042             setattr(self, attr, value)
2043         return getattr(self, attr)
2044
2045     # Force a checkpoint in fast-import and wait for it to finish
2046     def checkpoint(self):
2047         self.gitStream.write("checkpoint\n\n")
2048         self.gitStream.write("progress checkpoint\n\n")
2049         out = self.gitOutput.readline()
2050         if self.verbose:
2051             print "checkpoint finished: " + out
2052
2053     def extractFilesFromCommit(self, commit):
2054         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2055                              for path in self.cloneExclude]
2056         files = []
2057         fnum = 0
2058         while commit.has_key("depotFile%s" % fnum):
2059             path =  commit["depotFile%s" % fnum]
2060
2061             if [p for p in self.cloneExclude
2062                 if p4PathStartsWith(path, p)]:
2063                 found = False
2064             else:
2065                 found = [p for p in self.depotPaths
2066                          if p4PathStartsWith(path, p)]
2067             if not found:
2068                 fnum = fnum + 1
2069                 continue
2070
2071             file = {}
2072             file["path"] = path
2073             file["rev"] = commit["rev%s" % fnum]
2074             file["action"] = commit["action%s" % fnum]
2075             file["type"] = commit["type%s" % fnum]
2076             files.append(file)
2077             fnum = fnum + 1
2078         return files
2079
2080     def stripRepoPath(self, path, prefixes):
2081         """When streaming files, this is called to map a p4 depot path
2082            to where it should go in git.  The prefixes are either
2083            self.depotPaths, or self.branchPrefixes in the case of
2084            branch detection."""
2085
2086         if self.useClientSpec:
2087             # branch detection moves files up a level (the branch name)
2088             # from what client spec interpretation gives
2089             path = self.clientSpecDirs.map_in_client(path)
2090             if self.detectBranches:
2091                 for b in self.knownBranches:
2092                     if path.startswith(b + "/"):
2093                         path = path[len(b)+1:]
2094
2095         elif self.keepRepoPath:
2096             # Preserve everything in relative path name except leading
2097             # //depot/; just look at first prefix as they all should
2098             # be in the same depot.
2099             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2100             if p4PathStartsWith(path, depot):
2101                 path = path[len(depot):]
2102
2103         else:
2104             for p in prefixes:
2105                 if p4PathStartsWith(path, p):
2106                     path = path[len(p):]
2107                     break
2108
2109         path = wildcard_decode(path)
2110         return path
2111
2112     def splitFilesIntoBranches(self, commit):
2113         """Look at each depotFile in the commit to figure out to what
2114            branch it belongs."""
2115
2116         if self.clientSpecDirs:
2117             files = self.extractFilesFromCommit(commit)
2118             self.clientSpecDirs.update_client_spec_path_cache(files)
2119
2120         branches = {}
2121         fnum = 0
2122         while commit.has_key("depotFile%s" % fnum):
2123             path =  commit["depotFile%s" % fnum]
2124             found = [p for p in self.depotPaths
2125                      if p4PathStartsWith(path, p)]
2126             if not found:
2127                 fnum = fnum + 1
2128                 continue
2129
2130             file = {}
2131             file["path"] = path
2132             file["rev"] = commit["rev%s" % fnum]
2133             file["action"] = commit["action%s" % fnum]
2134             file["type"] = commit["type%s" % fnum]
2135             fnum = fnum + 1
2136
2137             # start with the full relative path where this file would
2138             # go in a p4 client
2139             if self.useClientSpec:
2140                 relPath = self.clientSpecDirs.map_in_client(path)
2141             else:
2142                 relPath = self.stripRepoPath(path, self.depotPaths)
2143
2144             for branch in self.knownBranches.keys():
2145                 # add a trailing slash so that a commit into qt/4.2foo
2146                 # doesn't end up in qt/4.2, e.g.
2147                 if relPath.startswith(branch + "/"):
2148                     if branch not in branches:
2149                         branches[branch] = []
2150                     branches[branch].append(file)
2151                     break
2152
2153         return branches
2154
2155     # output one file from the P4 stream
2156     # - helper for streamP4Files
2157
2158     def streamOneP4File(self, file, contents):
2159         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2160         if verbose:
2161             sys.stderr.write("%s\n" % relPath)
2162
2163         (type_base, type_mods) = split_p4_type(file["type"])
2164
2165         git_mode = "100644"
2166         if "x" in type_mods:
2167             git_mode = "100755"
2168         if type_base == "symlink":
2169             git_mode = "120000"
2170             # p4 print on a symlink sometimes contains "target\n";
2171             # if it does, remove the newline
2172             data = ''.join(contents)
2173             if not data:
2174                 # Some version of p4 allowed creating a symlink that pointed
2175                 # to nothing.  This causes p4 errors when checking out such
2176                 # a change, and errors here too.  Work around it by ignoring
2177                 # the bad symlink; hopefully a future change fixes it.
2178                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2179                 return
2180             elif data[-1] == '\n':
2181                 contents = [data[:-1]]
2182             else:
2183                 contents = [data]
2184
2185         if type_base == "utf16":
2186             # p4 delivers different text in the python output to -G
2187             # than it does when using "print -o", or normal p4 client
2188             # operations.  utf16 is converted to ascii or utf8, perhaps.
2189             # But ascii text saved as -t utf16 is completely mangled.
2190             # Invoke print -o to get the real contents.
2191             #
2192             # On windows, the newlines will always be mangled by print, so put
2193             # them back too.  This is not needed to the cygwin windows version,
2194             # just the native "NT" type.
2195             #
2196             text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
2197             if p4_version_string().find("/NT") >= 0:
2198                 text = text.replace("\r\n", "\n")
2199             contents = [ text ]
2200
2201         if type_base == "apple":
2202             # Apple filetype files will be streamed as a concatenation of
2203             # its appledouble header and the contents.  This is useless
2204             # on both macs and non-macs.  If using "print -q -o xx", it
2205             # will create "xx" with the data, and "%xx" with the header.
2206             # This is also not very useful.
2207             #
2208             # Ideally, someday, this script can learn how to generate
2209             # appledouble files directly and import those to git, but
2210             # non-mac machines can never find a use for apple filetype.
2211             print "\nIgnoring apple filetype file %s" % file['depotFile']
2212             return
2213
2214         # Note that we do not try to de-mangle keywords on utf16 files,
2215         # even though in theory somebody may want that.
2216         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2217         if pattern:
2218             regexp = re.compile(pattern, re.VERBOSE)
2219             text = ''.join(contents)
2220             text = regexp.sub(r'$\1$', text)
2221             contents = [ text ]
2222
2223         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2224
2225         # total length...
2226         length = 0
2227         for d in contents:
2228             length = length + len(d)
2229
2230         self.gitStream.write("data %d\n" % length)
2231         for d in contents:
2232             self.gitStream.write(d)
2233         self.gitStream.write("\n")
2234
2235     def streamOneP4Deletion(self, file):
2236         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2237         if verbose:
2238             sys.stderr.write("delete %s\n" % relPath)
2239         self.gitStream.write("D %s\n" % relPath)
2240
2241     # handle another chunk of streaming data
2242     def streamP4FilesCb(self, marshalled):
2243
2244         # catch p4 errors and complain
2245         err = None
2246         if "code" in marshalled:
2247             if marshalled["code"] == "error":
2248                 if "data" in marshalled:
2249                     err = marshalled["data"].rstrip()
2250         if err:
2251             f = None
2252             if self.stream_have_file_info:
2253                 if "depotFile" in self.stream_file:
2254                     f = self.stream_file["depotFile"]
2255             # force a failure in fast-import, else an empty
2256             # commit will be made
2257             self.gitStream.write("\n")
2258             self.gitStream.write("die-now\n")
2259             self.gitStream.close()
2260             # ignore errors, but make sure it exits first
2261             self.importProcess.wait()
2262             if f:
2263                 die("Error from p4 print for %s: %s" % (f, err))
2264             else:
2265                 die("Error from p4 print: %s" % err)
2266
2267         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2268             # start of a new file - output the old one first
2269             self.streamOneP4File(self.stream_file, self.stream_contents)
2270             self.stream_file = {}
2271             self.stream_contents = []
2272             self.stream_have_file_info = False
2273
2274         # pick up the new file information... for the
2275         # 'data' field we need to append to our array
2276         for k in marshalled.keys():
2277             if k == 'data':
2278                 self.stream_contents.append(marshalled['data'])
2279             else:
2280                 self.stream_file[k] = marshalled[k]
2281
2282         self.stream_have_file_info = True
2283
2284     # Stream directly from "p4 files" into "git fast-import"
2285     def streamP4Files(self, files):
2286         filesForCommit = []
2287         filesToRead = []
2288         filesToDelete = []
2289
2290         for f in files:
2291             # if using a client spec, only add the files that have
2292             # a path in the client
2293             if self.clientSpecDirs:
2294                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2295                     continue
2296
2297             filesForCommit.append(f)
2298             if f['action'] in self.delete_actions:
2299                 filesToDelete.append(f)
2300             else:
2301                 filesToRead.append(f)
2302
2303         # deleted files...
2304         for f in filesToDelete:
2305             self.streamOneP4Deletion(f)
2306
2307         if len(filesToRead) > 0:
2308             self.stream_file = {}
2309             self.stream_contents = []
2310             self.stream_have_file_info = False
2311
2312             # curry self argument
2313             def streamP4FilesCbSelf(entry):
2314                 self.streamP4FilesCb(entry)
2315
2316             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2317
2318             p4CmdList(["-x", "-", "print"],
2319                       stdin=fileArgs,
2320                       cb=streamP4FilesCbSelf)
2321
2322             # do the last chunk
2323             if self.stream_file.has_key('depotFile'):
2324                 self.streamOneP4File(self.stream_file, self.stream_contents)
2325
2326     def make_email(self, userid):
2327         if userid in self.users:
2328             return self.users[userid]
2329         else:
2330             return "%s <a@b>" % userid
2331
2332     # Stream a p4 tag
2333     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2334         if verbose:
2335             print "writing tag %s for commit %s" % (labelName, commit)
2336         gitStream.write("tag %s\n" % labelName)
2337         gitStream.write("from %s\n" % commit)
2338
2339         if labelDetails.has_key('Owner'):
2340             owner = labelDetails["Owner"]
2341         else:
2342             owner = None
2343
2344         # Try to use the owner of the p4 label, or failing that,
2345         # the current p4 user id.
2346         if owner:
2347             email = self.make_email(owner)
2348         else:
2349             email = self.make_email(self.p4UserId())
2350         tagger = "%s %s %s" % (email, epoch, self.tz)
2351
2352         gitStream.write("tagger %s\n" % tagger)
2353
2354         print "labelDetails=",labelDetails
2355         if labelDetails.has_key('Description'):
2356             description = labelDetails['Description']
2357         else:
2358             description = 'Label from git p4'
2359
2360         gitStream.write("data %d\n" % len(description))
2361         gitStream.write(description)
2362         gitStream.write("\n")
2363
2364     def commit(self, details, files, branch, parent = ""):
2365         epoch = details["time"]
2366         author = details["user"]
2367
2368         if self.verbose:
2369             print "commit into %s" % branch
2370
2371         # start with reading files; if that fails, we should not
2372         # create a commit.
2373         new_files = []
2374         for f in files:
2375             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2376                 new_files.append (f)
2377             else:
2378                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2379
2380         if self.clientSpecDirs:
2381             self.clientSpecDirs.update_client_spec_path_cache(files)
2382
2383         self.gitStream.write("commit %s\n" % branch)
2384 #        gitStream.write("mark :%s\n" % details["change"])
2385         self.committedChanges.add(int(details["change"]))
2386         committer = ""
2387         if author not in self.users:
2388             self.getUserMapFromPerforceServer()
2389         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2390
2391         self.gitStream.write("committer %s\n" % committer)
2392
2393         self.gitStream.write("data <<EOT\n")
2394         self.gitStream.write(details["desc"])
2395         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2396                              (','.join(self.branchPrefixes), details["change"]))
2397         if len(details['options']) > 0:
2398             self.gitStream.write(": options = %s" % details['options'])
2399         self.gitStream.write("]\nEOT\n\n")
2400
2401         if len(parent) > 0:
2402             if self.verbose:
2403                 print "parent %s" % parent
2404             self.gitStream.write("from %s\n" % parent)
2405
2406         self.streamP4Files(new_files)
2407         self.gitStream.write("\n")
2408
2409         change = int(details["change"])
2410
2411         if self.labels.has_key(change):
2412             label = self.labels[change]
2413             labelDetails = label[0]
2414             labelRevisions = label[1]
2415             if self.verbose:
2416                 print "Change %s is labelled %s" % (change, labelDetails)
2417
2418             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2419                                                 for p in self.branchPrefixes])
2420
2421             if len(files) == len(labelRevisions):
2422
2423                 cleanedFiles = {}
2424                 for info in files:
2425                     if info["action"] in self.delete_actions:
2426                         continue
2427                     cleanedFiles[info["depotFile"]] = info["rev"]
2428
2429                 if cleanedFiles == labelRevisions:
2430                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2431
2432                 else:
2433                     if not self.silent:
2434                         print ("Tag %s does not match with change %s: files do not match."
2435                                % (labelDetails["label"], change))
2436
2437             else:
2438                 if not self.silent:
2439                     print ("Tag %s does not match with change %s: file count is different."
2440                            % (labelDetails["label"], change))
2441
2442     # Build a dictionary of changelists and labels, for "detect-labels" option.
2443     def getLabels(self):
2444         self.labels = {}
2445
2446         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2447         if len(l) > 0 and not self.silent:
2448             print "Finding files belonging to labels in %s" % `self.depotPaths`
2449
2450         for output in l:
2451             label = output["label"]
2452             revisions = {}
2453             newestChange = 0
2454             if self.verbose:
2455                 print "Querying files for label %s" % label
2456             for file in p4CmdList(["files"] +
2457                                       ["%s...@%s" % (p, label)
2458                                           for p in self.depotPaths]):
2459                 revisions[file["depotFile"]] = file["rev"]
2460                 change = int(file["change"])
2461                 if change > newestChange:
2462                     newestChange = change
2463
2464             self.labels[newestChange] = [output, revisions]
2465
2466         if self.verbose:
2467             print "Label changes: %s" % self.labels.keys()
2468
2469     # Import p4 labels as git tags. A direct mapping does not
2470     # exist, so assume that if all the files are at the same revision
2471     # then we can use that, or it's something more complicated we should
2472     # just ignore.
2473     def importP4Labels(self, stream, p4Labels):
2474         if verbose:
2475             print "import p4 labels: " + ' '.join(p4Labels)
2476
2477         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2478         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2479         if len(validLabelRegexp) == 0:
2480             validLabelRegexp = defaultLabelRegexp
2481         m = re.compile(validLabelRegexp)
2482
2483         for name in p4Labels:
2484             commitFound = False
2485
2486             if not m.match(name):
2487                 if verbose:
2488                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2489                 continue
2490
2491             if name in ignoredP4Labels:
2492                 continue
2493
2494             labelDetails = p4CmdList(['label', "-o", name])[0]
2495
2496             # get the most recent changelist for each file in this label
2497             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2498                                 for p in self.depotPaths])
2499
2500             if change.has_key('change'):
2501                 # find the corresponding git commit; take the oldest commit
2502                 changelist = int(change['change'])
2503                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2504                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2505                 if len(gitCommit) == 0:
2506                     print "could not find git commit for changelist %d" % changelist
2507                 else:
2508                     gitCommit = gitCommit.strip()
2509                     commitFound = True
2510                     # Convert from p4 time format
2511                     try:
2512                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2513                     except ValueError:
2514                         print "Could not convert label time %s" % labelDetails['Update']
2515                         tmwhen = 1
2516
2517                     when = int(time.mktime(tmwhen))
2518                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2519                     if verbose:
2520                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2521             else:
2522                 if verbose:
2523                     print "Label %s has no changelists - possibly deleted?" % name
2524
2525             if not commitFound:
2526                 # We can't import this label; don't try again as it will get very
2527                 # expensive repeatedly fetching all the files for labels that will
2528                 # never be imported. If the label is moved in the future, the
2529                 # ignore will need to be removed manually.
2530                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2531
2532     def guessProjectName(self):
2533         for p in self.depotPaths:
2534             if p.endswith("/"):
2535                 p = p[:-1]
2536             p = p[p.strip().rfind("/") + 1:]
2537             if not p.endswith("/"):
2538                p += "/"
2539             return p
2540
2541     def getBranchMapping(self):
2542         lostAndFoundBranches = set()
2543
2544         user = gitConfig("git-p4.branchUser")
2545         if len(user) > 0:
2546             command = "branches -u %s" % user
2547         else:
2548             command = "branches"
2549
2550         for info in p4CmdList(command):
2551             details = p4Cmd(["branch", "-o", info["branch"]])
2552             viewIdx = 0
2553             while details.has_key("View%s" % viewIdx):
2554                 paths = details["View%s" % viewIdx].split(" ")
2555                 viewIdx = viewIdx + 1
2556                 # require standard //depot/foo/... //depot/bar/... mapping
2557                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2558                     continue
2559                 source = paths[0]
2560                 destination = paths[1]
2561                 ## HACK
2562                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2563                     source = source[len(self.depotPaths[0]):-4]
2564                     destination = destination[len(self.depotPaths[0]):-4]
2565
2566                     if destination in self.knownBranches:
2567                         if not self.silent:
2568                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2569                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2570                         continue
2571
2572                     self.knownBranches[destination] = source
2573
2574                     lostAndFoundBranches.discard(destination)
2575
2576                     if source not in self.knownBranches:
2577                         lostAndFoundBranches.add(source)
2578
2579         # Perforce does not strictly require branches to be defined, so we also
2580         # check git config for a branch list.
2581         #
2582         # Example of branch definition in git config file:
2583         # [git-p4]
2584         #   branchList=main:branchA
2585         #   branchList=main:branchB
2586         #   branchList=branchA:branchC
2587         configBranches = gitConfigList("git-p4.branchList")
2588         for branch in configBranches:
2589             if branch:
2590                 (source, destination) = branch.split(":")
2591                 self.knownBranches[destination] = source
2592
2593                 lostAndFoundBranches.discard(destination)
2594
2595                 if source not in self.knownBranches:
2596                     lostAndFoundBranches.add(source)
2597
2598
2599         for branch in lostAndFoundBranches:
2600             self.knownBranches[branch] = branch
2601
2602     def getBranchMappingFromGitBranches(self):
2603         branches = p4BranchesInGit(self.importIntoRemotes)
2604         for branch in branches.keys():
2605             if branch == "master":
2606                 branch = "main"
2607             else:
2608                 branch = branch[len(self.projectName):]
2609             self.knownBranches[branch] = branch
2610
2611     def updateOptionDict(self, d):
2612         option_keys = {}
2613         if self.keepRepoPath:
2614             option_keys['keepRepoPath'] = 1
2615
2616         d["options"] = ' '.join(sorted(option_keys.keys()))
2617
2618     def readOptions(self, d):
2619         self.keepRepoPath = (d.has_key('options')
2620                              and ('keepRepoPath' in d['options']))
2621
2622     def gitRefForBranch(self, branch):
2623         if branch == "main":
2624             return self.refPrefix + "master"
2625
2626         if len(branch) <= 0:
2627             return branch
2628
2629         return self.refPrefix + self.projectName + branch
2630
2631     def gitCommitByP4Change(self, ref, change):
2632         if self.verbose:
2633             print "looking in ref " + ref + " for change %s using bisect..." % change
2634
2635         earliestCommit = ""
2636         latestCommit = parseRevision(ref)
2637
2638         while True:
2639             if self.verbose:
2640                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2641             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2642             if len(next) == 0:
2643                 if self.verbose:
2644                     print "argh"
2645                 return ""
2646             log = extractLogMessageFromGitCommit(next)
2647             settings = extractSettingsGitLog(log)
2648             currentChange = int(settings['change'])
2649             if self.verbose:
2650                 print "current change %s" % currentChange
2651
2652             if currentChange == change:
2653                 if self.verbose:
2654                     print "found %s" % next
2655                 return next
2656
2657             if currentChange < change:
2658                 earliestCommit = "^%s" % next
2659             else:
2660                 latestCommit = "%s" % next
2661
2662         return ""
2663
2664     def importNewBranch(self, branch, maxChange):
2665         # make fast-import flush all changes to disk and update the refs using the checkpoint
2666         # command so that we can try to find the branch parent in the git history
2667         self.gitStream.write("checkpoint\n\n");
2668         self.gitStream.flush();
2669         branchPrefix = self.depotPaths[0] + branch + "/"
2670         range = "@1,%s" % maxChange
2671         #print "prefix" + branchPrefix
2672         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2673         if len(changes) <= 0:
2674             return False
2675         firstChange = changes[0]
2676         #print "first change in branch: %s" % firstChange
2677         sourceBranch = self.knownBranches[branch]
2678         sourceDepotPath = self.depotPaths[0] + sourceBranch
2679         sourceRef = self.gitRefForBranch(sourceBranch)
2680         #print "source " + sourceBranch
2681
2682         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2683         #print "branch parent: %s" % branchParentChange
2684         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2685         if len(gitParent) > 0:
2686             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2687             #print "parent git commit: %s" % gitParent
2688
2689         self.importChanges(changes)
2690         return True
2691
2692     def searchParent(self, parent, branch, target):
2693         parentFound = False
2694         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2695                                      "--no-merges", parent]):
2696             blob = blob.strip()
2697             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2698                 parentFound = True
2699                 if self.verbose:
2700                     print "Found parent of %s in commit %s" % (branch, blob)
2701                 break
2702         if parentFound:
2703             return blob
2704         else:
2705             return None
2706
2707     def importChanges(self, changes):
2708         cnt = 1
2709         for change in changes:
2710             description = p4_describe(change)
2711             self.updateOptionDict(description)
2712
2713             if not self.silent:
2714                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2715                 sys.stdout.flush()
2716             cnt = cnt + 1
2717
2718             try:
2719                 if self.detectBranches:
2720                     branches = self.splitFilesIntoBranches(description)
2721                     for branch in branches.keys():
2722                         ## HACK  --hwn
2723                         branchPrefix = self.depotPaths[0] + branch + "/"
2724                         self.branchPrefixes = [ branchPrefix ]
2725
2726                         parent = ""
2727
2728                         filesForCommit = branches[branch]
2729
2730                         if self.verbose:
2731                             print "branch is %s" % branch
2732
2733                         self.updatedBranches.add(branch)
2734
2735                         if branch not in self.createdBranches:
2736                             self.createdBranches.add(branch)
2737                             parent = self.knownBranches[branch]
2738                             if parent == branch:
2739                                 parent = ""
2740                             else:
2741                                 fullBranch = self.projectName + branch
2742                                 if fullBranch not in self.p4BranchesInGit:
2743                                     if not self.silent:
2744                                         print("\n    Importing new branch %s" % fullBranch);
2745                                     if self.importNewBranch(branch, change - 1):
2746                                         parent = ""
2747                                         self.p4BranchesInGit.append(fullBranch)
2748                                     if not self.silent:
2749                                         print("\n    Resuming with change %s" % change);
2750
2751                                 if self.verbose:
2752                                     print "parent determined through known branches: %s" % parent
2753
2754                         branch = self.gitRefForBranch(branch)
2755                         parent = self.gitRefForBranch(parent)
2756
2757                         if self.verbose:
2758                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2759
2760                         if len(parent) == 0 and branch in self.initialParents:
2761                             parent = self.initialParents[branch]
2762                             del self.initialParents[branch]
2763
2764                         blob = None
2765                         if len(parent) > 0:
2766                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2767                             if self.verbose:
2768                                 print "Creating temporary branch: " + tempBranch
2769                             self.commit(description, filesForCommit, tempBranch)
2770                             self.tempBranches.append(tempBranch)
2771                             self.checkpoint()
2772                             blob = self.searchParent(parent, branch, tempBranch)
2773                         if blob:
2774                             self.commit(description, filesForCommit, branch, blob)
2775                         else:
2776                             if self.verbose:
2777                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2778                             self.commit(description, filesForCommit, branch, parent)
2779                 else:
2780                     files = self.extractFilesFromCommit(description)
2781                     self.commit(description, files, self.branch,
2782                                 self.initialParent)
2783                     # only needed once, to connect to the previous commit
2784                     self.initialParent = ""
2785             except IOError:
2786                 print self.gitError.read()
2787                 sys.exit(1)
2788
2789     def importHeadRevision(self, revision):
2790         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2791
2792         details = {}
2793         details["user"] = "git perforce import user"
2794         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2795                            % (' '.join(self.depotPaths), revision))
2796         details["change"] = revision
2797         newestRevision = 0
2798
2799         fileCnt = 0
2800         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2801
2802         for info in p4CmdList(["files"] + fileArgs):
2803
2804             if 'code' in info and info['code'] == 'error':
2805                 sys.stderr.write("p4 returned an error: %s\n"
2806                                  % info['data'])
2807                 if info['data'].find("must refer to client") >= 0:
2808                     sys.stderr.write("This particular p4 error is misleading.\n")
2809                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2810                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2811                 sys.exit(1)
2812             if 'p4ExitCode' in info:
2813                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2814                 sys.exit(1)
2815
2816
2817             change = int(info["change"])
2818             if change > newestRevision:
2819                 newestRevision = change
2820
2821             if info["action"] in self.delete_actions:
2822                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2823                 #fileCnt = fileCnt + 1
2824                 continue
2825
2826             for prop in ["depotFile", "rev", "action", "type" ]:
2827                 details["%s%s" % (prop, fileCnt)] = info[prop]
2828
2829             fileCnt = fileCnt + 1
2830
2831         details["change"] = newestRevision
2832
2833         # Use time from top-most change so that all git p4 clones of
2834         # the same p4 repo have the same commit SHA1s.
2835         res = p4_describe(newestRevision)
2836         details["time"] = res["time"]
2837
2838         self.updateOptionDict(details)
2839         try:
2840             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2841         except IOError:
2842             print "IO error with git fast-import. Is your git version recent enough?"
2843             print self.gitError.read()
2844
2845
2846     def run(self, args):
2847         self.depotPaths = []
2848         self.changeRange = ""
2849         self.previousDepotPaths = []
2850         self.hasOrigin = False
2851
2852         # map from branch depot path to parent branch
2853         self.knownBranches = {}
2854         self.initialParents = {}
2855
2856         if self.importIntoRemotes:
2857             self.refPrefix = "refs/remotes/p4/"
2858         else:
2859             self.refPrefix = "refs/heads/p4/"
2860
2861         if self.syncWithOrigin:
2862             self.hasOrigin = originP4BranchesExist()
2863             if self.hasOrigin:
2864                 if not self.silent:
2865                     print 'Syncing with origin first, using "git fetch origin"'
2866                 system("git fetch origin")
2867
2868         branch_arg_given = bool(self.branch)
2869         if len(self.branch) == 0:
2870             self.branch = self.refPrefix + "master"
2871             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2872                 system("git update-ref %s refs/heads/p4" % self.branch)
2873                 system("git branch -D p4")
2874
2875         # accept either the command-line option, or the configuration variable
2876         if self.useClientSpec:
2877             # will use this after clone to set the variable
2878             self.useClientSpec_from_options = True
2879         else:
2880             if gitConfigBool("git-p4.useclientspec"):
2881                 self.useClientSpec = True
2882         if self.useClientSpec:
2883             self.clientSpecDirs = getClientSpec()
2884
2885         # TODO: should always look at previous commits,
2886         # merge with previous imports, if possible.
2887         if args == []:
2888             if self.hasOrigin:
2889                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2890
2891             # branches holds mapping from branch name to sha1
2892             branches = p4BranchesInGit(self.importIntoRemotes)
2893
2894             # restrict to just this one, disabling detect-branches
2895             if branch_arg_given:
2896                 short = self.branch.split("/")[-1]
2897                 if short in branches:
2898                     self.p4BranchesInGit = [ short ]
2899             else:
2900                 self.p4BranchesInGit = branches.keys()
2901
2902             if len(self.p4BranchesInGit) > 1:
2903                 if not self.silent:
2904                     print "Importing from/into multiple branches"
2905                 self.detectBranches = True
2906                 for branch in branches.keys():
2907                     self.initialParents[self.refPrefix + branch] = \
2908                         branches[branch]
2909
2910             if self.verbose:
2911                 print "branches: %s" % self.p4BranchesInGit
2912
2913             p4Change = 0
2914             for branch in self.p4BranchesInGit:
2915                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2916
2917                 settings = extractSettingsGitLog(logMsg)
2918
2919                 self.readOptions(settings)
2920                 if (settings.has_key('depot-paths')
2921                     and settings.has_key ('change')):
2922                     change = int(settings['change']) + 1
2923                     p4Change = max(p4Change, change)
2924
2925                     depotPaths = sorted(settings['depot-paths'])
2926                     if self.previousDepotPaths == []:
2927                         self.previousDepotPaths = depotPaths
2928                     else:
2929                         paths = []
2930                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2931                             prev_list = prev.split("/")
2932                             cur_list = cur.split("/")
2933                             for i in range(0, min(len(cur_list), len(prev_list))):
2934                                 if cur_list[i] <> prev_list[i]:
2935                                     i = i - 1
2936                                     break
2937
2938                             paths.append ("/".join(cur_list[:i + 1]))
2939
2940                         self.previousDepotPaths = paths
2941
2942             if p4Change > 0:
2943                 self.depotPaths = sorted(self.previousDepotPaths)
2944                 self.changeRange = "@%s,#head" % p4Change
2945                 if not self.silent and not self.detectBranches:
2946                     print "Performing incremental import into %s git branch" % self.branch
2947
2948         # accept multiple ref name abbreviations:
2949         #    refs/foo/bar/branch -> use it exactly
2950         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2951         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2952         if not self.branch.startswith("refs/"):
2953             if self.importIntoRemotes:
2954                 prepend = "refs/remotes/"
2955             else:
2956                 prepend = "refs/heads/"
2957             if not self.branch.startswith("p4/"):
2958                 prepend += "p4/"
2959             self.branch = prepend + self.branch
2960
2961         if len(args) == 0 and self.depotPaths:
2962             if not self.silent:
2963                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2964         else:
2965             if self.depotPaths and self.depotPaths != args:
2966                 print ("previous import used depot path %s and now %s was specified. "
2967                        "This doesn't work!" % (' '.join (self.depotPaths),
2968                                                ' '.join (args)))
2969                 sys.exit(1)
2970
2971             self.depotPaths = sorted(args)
2972
2973         revision = ""
2974         self.users = {}
2975
2976         # Make sure no revision specifiers are used when --changesfile
2977         # is specified.
2978         bad_changesfile = False
2979         if len(self.changesFile) > 0:
2980             for p in self.depotPaths:
2981                 if p.find("@") >= 0 or p.find("#") >= 0:
2982                     bad_changesfile = True
2983                     break
2984         if bad_changesfile:
2985             die("Option --changesfile is incompatible with revision specifiers")
2986
2987         newPaths = []
2988         for p in self.depotPaths:
2989             if p.find("@") != -1:
2990                 atIdx = p.index("@")
2991                 self.changeRange = p[atIdx:]
2992                 if self.changeRange == "@all":
2993                     self.changeRange = ""
2994                 elif ',' not in self.changeRange:
2995                     revision = self.changeRange
2996                     self.changeRange = ""
2997                 p = p[:atIdx]
2998             elif p.find("#") != -1:
2999                 hashIdx = p.index("#")
3000                 revision = p[hashIdx:]
3001                 p = p[:hashIdx]
3002             elif self.previousDepotPaths == []:
3003                 # pay attention to changesfile, if given, else import
3004                 # the entire p4 tree at the head revision
3005                 if len(self.changesFile) == 0:
3006                     revision = "#head"
3007
3008             p = re.sub ("\.\.\.$", "", p)
3009             if not p.endswith("/"):
3010                 p += "/"
3011
3012             newPaths.append(p)
3013
3014         self.depotPaths = newPaths
3015
3016         # --detect-branches may change this for each branch
3017         self.branchPrefixes = self.depotPaths
3018
3019         self.loadUserMapFromCache()
3020         self.labels = {}
3021         if self.detectLabels:
3022             self.getLabels();
3023
3024         if self.detectBranches:
3025             ## FIXME - what's a P4 projectName ?
3026             self.projectName = self.guessProjectName()
3027
3028             if self.hasOrigin:
3029                 self.getBranchMappingFromGitBranches()
3030             else:
3031                 self.getBranchMapping()
3032             if self.verbose:
3033                 print "p4-git branches: %s" % self.p4BranchesInGit
3034                 print "initial parents: %s" % self.initialParents
3035             for b in self.p4BranchesInGit:
3036                 if b != "master":
3037
3038                     ## FIXME
3039                     b = b[len(self.projectName):]
3040                 self.createdBranches.add(b)
3041
3042         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3043
3044         self.importProcess = subprocess.Popen(["git", "fast-import"],
3045                                               stdin=subprocess.PIPE,
3046                                               stdout=subprocess.PIPE,
3047                                               stderr=subprocess.PIPE);
3048         self.gitOutput = self.importProcess.stdout
3049         self.gitStream = self.importProcess.stdin
3050         self.gitError = self.importProcess.stderr
3051
3052         if revision:
3053             self.importHeadRevision(revision)
3054         else:
3055             changes = []
3056
3057             if len(self.changesFile) > 0:
3058                 output = open(self.changesFile).readlines()
3059                 changeSet = set()
3060                 for line in output:
3061                     changeSet.add(int(line))
3062
3063                 for change in changeSet:
3064                     changes.append(change)
3065
3066                 changes.sort()
3067             else:
3068                 # catch "git p4 sync" with no new branches, in a repo that
3069                 # does not have any existing p4 branches
3070                 if len(args) == 0:
3071                     if not self.p4BranchesInGit:
3072                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3073
3074                     # The default branch is master, unless --branch is used to
3075                     # specify something else.  Make sure it exists, or complain
3076                     # nicely about how to use --branch.
3077                     if not self.detectBranches:
3078                         if not branch_exists(self.branch):
3079                             if branch_arg_given:
3080                                 die("Error: branch %s does not exist." % self.branch)
3081                             else:
3082                                 die("Error: no branch %s; perhaps specify one with --branch." %
3083                                     self.branch)
3084
3085                 if self.verbose:
3086                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3087                                                               self.changeRange)
3088                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3089
3090                 if len(self.maxChanges) > 0:
3091                     changes = changes[:min(int(self.maxChanges), len(changes))]
3092
3093             if len(changes) == 0:
3094                 if not self.silent:
3095                     print "No changes to import!"
3096             else:
3097                 if not self.silent and not self.detectBranches:
3098                     print "Import destination: %s" % self.branch
3099
3100                 self.updatedBranches = set()
3101
3102                 if not self.detectBranches:
3103                     if args:
3104                         # start a new branch
3105                         self.initialParent = ""
3106                     else:
3107                         # build on a previous revision
3108                         self.initialParent = parseRevision(self.branch)
3109
3110                 self.importChanges(changes)
3111
3112                 if not self.silent:
3113                     print ""
3114                     if len(self.updatedBranches) > 0:
3115                         sys.stdout.write("Updated branches: ")
3116                         for b in self.updatedBranches:
3117                             sys.stdout.write("%s " % b)
3118                         sys.stdout.write("\n")
3119
3120         if gitConfigBool("git-p4.importLabels"):
3121             self.importLabels = True
3122
3123         if self.importLabels:
3124             p4Labels = getP4Labels(self.depotPaths)
3125             gitTags = getGitTags()
3126
3127             missingP4Labels = p4Labels - gitTags
3128             self.importP4Labels(self.gitStream, missingP4Labels)
3129
3130         self.gitStream.close()
3131         if self.importProcess.wait() != 0:
3132             die("fast-import failed: %s" % self.gitError.read())
3133         self.gitOutput.close()
3134         self.gitError.close()
3135
3136         # Cleanup temporary branches created during import
3137         if self.tempBranches != []:
3138             for branch in self.tempBranches:
3139                 read_pipe("git update-ref -d %s" % branch)
3140             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3141
3142         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3143         # a convenient shortcut refname "p4".
3144         if self.importIntoRemotes:
3145             head_ref = self.refPrefix + "HEAD"
3146             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3147                 system(["git", "symbolic-ref", head_ref, self.branch])
3148
3149         return True
3150
3151 class P4Rebase(Command):
3152     def __init__(self):
3153         Command.__init__(self)
3154         self.options = [
3155                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3156         ]
3157         self.importLabels = False
3158         self.description = ("Fetches the latest revision from perforce and "
3159                             + "rebases the current work (branch) against it")
3160
3161     def run(self, args):
3162         sync = P4Sync()
3163         sync.importLabels = self.importLabels
3164         sync.run([])
3165
3166         return self.rebase()
3167
3168     def rebase(self):
3169         if os.system("git update-index --refresh") != 0:
3170             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3171         if len(read_pipe("git diff-index HEAD --")) > 0:
3172             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3173
3174         [upstream, settings] = findUpstreamBranchPoint()
3175         if len(upstream) == 0:
3176             die("Cannot find upstream branchpoint for rebase")
3177
3178         # the branchpoint may be p4/foo~3, so strip off the parent
3179         upstream = re.sub("~[0-9]+$", "", upstream)
3180
3181         print "Rebasing the current branch onto %s" % upstream
3182         oldHead = read_pipe("git rev-parse HEAD").strip()
3183         system("git rebase %s" % upstream)
3184         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3185         return True
3186
3187 class P4Clone(P4Sync):
3188     def __init__(self):
3189         P4Sync.__init__(self)
3190         self.description = "Creates a new git repository and imports from Perforce into it"
3191         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3192         self.options += [
3193             optparse.make_option("--destination", dest="cloneDestination",
3194                                  action='store', default=None,
3195                                  help="where to leave result of the clone"),
3196             optparse.make_option("--bare", dest="cloneBare",
3197                                  action="store_true", default=False),
3198         ]
3199         self.cloneDestination = None
3200         self.needsGit = False
3201         self.cloneBare = False
3202
3203     def defaultDestination(self, args):
3204         ## TODO: use common prefix of args?
3205         depotPath = args[0]
3206         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3207         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3208         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3209         depotDir = re.sub(r"/$", "", depotDir)
3210         return os.path.split(depotDir)[1]
3211
3212     def run(self, args):
3213         if len(args) < 1:
3214             return False
3215
3216         if self.keepRepoPath and not self.cloneDestination:
3217             sys.stderr.write("Must specify destination for --keep-path\n")
3218             sys.exit(1)
3219
3220         depotPaths = args
3221
3222         if not self.cloneDestination and len(depotPaths) > 1:
3223             self.cloneDestination = depotPaths[-1]
3224             depotPaths = depotPaths[:-1]
3225
3226         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3227         for p in depotPaths:
3228             if not p.startswith("//"):
3229                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3230                 return False
3231
3232         if not self.cloneDestination:
3233             self.cloneDestination = self.defaultDestination(args)
3234
3235         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3236
3237         if not os.path.exists(self.cloneDestination):
3238             os.makedirs(self.cloneDestination)
3239         chdir(self.cloneDestination)
3240
3241         init_cmd = [ "git", "init" ]
3242         if self.cloneBare:
3243             init_cmd.append("--bare")
3244         retcode = subprocess.call(init_cmd)
3245         if retcode:
3246             raise CalledProcessError(retcode, init_cmd)
3247
3248         if not P4Sync.run(self, depotPaths):
3249             return False
3250
3251         # create a master branch and check out a work tree
3252         if gitBranchExists(self.branch):
3253             system([ "git", "branch", "master", self.branch ])
3254             if not self.cloneBare:
3255                 system([ "git", "checkout", "-f" ])
3256         else:
3257             print 'Not checking out any branch, use ' \
3258                   '"git checkout -q -b master <branch>"'
3259
3260         # auto-set this variable if invoked with --use-client-spec
3261         if self.useClientSpec_from_options:
3262             system("git config --bool git-p4.useclientspec true")
3263
3264         return True
3265
3266 class P4Branches(Command):
3267     def __init__(self):
3268         Command.__init__(self)
3269         self.options = [ ]
3270         self.description = ("Shows the git branches that hold imports and their "
3271                             + "corresponding perforce depot paths")
3272         self.verbose = False
3273
3274     def run(self, args):
3275         if originP4BranchesExist():
3276             createOrUpdateBranchesFromOrigin()
3277
3278         cmdline = "git rev-parse --symbolic "
3279         cmdline += " --remotes"
3280
3281         for line in read_pipe_lines(cmdline):
3282             line = line.strip()
3283
3284             if not line.startswith('p4/') or line == "p4/HEAD":
3285                 continue
3286             branch = line
3287
3288             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3289             settings = extractSettingsGitLog(log)
3290
3291             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3292         return True
3293
3294 class HelpFormatter(optparse.IndentedHelpFormatter):
3295     def __init__(self):
3296         optparse.IndentedHelpFormatter.__init__(self)
3297
3298     def format_description(self, description):
3299         if description:
3300             return description + "\n"
3301         else:
3302             return ""
3303
3304 def printUsage(commands):
3305     print "usage: %s <command> [options]" % sys.argv[0]
3306     print ""
3307     print "valid commands: %s" % ", ".join(commands)
3308     print ""
3309     print "Try %s <command> --help for command specific help." % sys.argv[0]
3310     print ""
3311
3312 commands = {
3313     "debug" : P4Debug,
3314     "submit" : P4Submit,
3315     "commit" : P4Submit,
3316     "sync" : P4Sync,
3317     "rebase" : P4Rebase,
3318     "clone" : P4Clone,
3319     "rollback" : P4RollBack,
3320     "branches" : P4Branches
3321 }
3322
3323
3324 def main():
3325     if len(sys.argv[1:]) == 0:
3326         printUsage(commands.keys())
3327         sys.exit(2)
3328
3329     cmdName = sys.argv[1]
3330     try:
3331         klass = commands[cmdName]
3332         cmd = klass()
3333     except KeyError:
3334         print "unknown command %s" % cmdName
3335         print ""
3336         printUsage(commands.keys())
3337         sys.exit(2)
3338
3339     options = cmd.options
3340     cmd.gitdir = os.environ.get("GIT_DIR", None)
3341
3342     args = sys.argv[2:]
3343
3344     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3345     if cmd.needsGit:
3346         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3347
3348     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3349                                    options,
3350                                    description = cmd.description,
3351                                    formatter = HelpFormatter())
3352
3353     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3354     global verbose
3355     verbose = cmd.verbose
3356     if cmd.needsGit:
3357         if cmd.gitdir == None:
3358             cmd.gitdir = os.path.abspath(".git")
3359             if not isValidGitDir(cmd.gitdir):
3360                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3361                 if os.path.exists(cmd.gitdir):
3362                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3363                     if len(cdup) > 0:
3364                         chdir(cdup);
3365
3366         if not isValidGitDir(cmd.gitdir):
3367             if isValidGitDir(cmd.gitdir + "/.git"):
3368                 cmd.gitdir += "/.git"
3369             else:
3370                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3371
3372         os.environ["GIT_DIR"] = cmd.gitdir
3373
3374     if not cmd.run(args):
3375         parser.print_help()
3376         sys.exit(2)
3377
3378
3379 if __name__ == '__main__':
3380     main()