Merge branch 'jk/strbuf-detach-always-non-null'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 import re, shutil
14
15 verbose = False
16
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
19
20 def p4_build_cmd(cmd):
21     """Build a suitable p4 command line.
22
23     This consolidates building and returning a p4 command line into one
24     location. It means that hooking into the environment, or other configuration
25     can be done more easily.
26     """
27     real_cmd = ["p4"]
28
29     user = gitConfig("git-p4.user")
30     if len(user) > 0:
31         real_cmd += ["-u",user]
32
33     password = gitConfig("git-p4.password")
34     if len(password) > 0:
35         real_cmd += ["-P", password]
36
37     port = gitConfig("git-p4.port")
38     if len(port) > 0:
39         real_cmd += ["-p", port]
40
41     host = gitConfig("git-p4.host")
42     if len(host) > 0:
43         real_cmd += ["-H", host]
44
45     client = gitConfig("git-p4.client")
46     if len(client) > 0:
47         real_cmd += ["-c", client]
48
49
50     if isinstance(cmd,basestring):
51         real_cmd = ' '.join(real_cmd) + ' ' + cmd
52     else:
53         real_cmd += cmd
54     return real_cmd
55
56 def chdir(dir):
57     # P4 uses the PWD environment variable rather than getcwd(). Since we're
58     # not using the shell, we have to set it ourselves.  This path could
59     # be relative, so go there first, then figure out where we ended up.
60     os.chdir(dir)
61     os.environ['PWD'] = os.getcwd()
62
63 def die(msg):
64     if verbose:
65         raise Exception(msg)
66     else:
67         sys.stderr.write(msg + "\n")
68         sys.exit(1)
69
70 def write_pipe(c, stdin):
71     if verbose:
72         sys.stderr.write('Writing pipe: %s\n' % str(c))
73
74     expand = isinstance(c,basestring)
75     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
76     pipe = p.stdin
77     val = pipe.write(stdin)
78     pipe.close()
79     if p.wait():
80         die('Command failed: %s' % str(c))
81
82     return val
83
84 def p4_write_pipe(c, stdin):
85     real_cmd = p4_build_cmd(c)
86     return write_pipe(real_cmd, stdin)
87
88 def read_pipe(c, ignore_error=False):
89     if verbose:
90         sys.stderr.write('Reading pipe: %s\n' % str(c))
91
92     expand = isinstance(c,basestring)
93     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
94     pipe = p.stdout
95     val = pipe.read()
96     if p.wait() and not ignore_error:
97         die('Command failed: %s' % str(c))
98
99     return val
100
101 def p4_read_pipe(c, ignore_error=False):
102     real_cmd = p4_build_cmd(c)
103     return read_pipe(real_cmd, ignore_error)
104
105 def read_pipe_lines(c):
106     if verbose:
107         sys.stderr.write('Reading pipe: %s\n' % str(c))
108
109     expand = isinstance(c, basestring)
110     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
111     pipe = p.stdout
112     val = pipe.readlines()
113     if pipe.close() or p.wait():
114         die('Command failed: %s' % str(c))
115
116     return val
117
118 def p4_read_pipe_lines(c):
119     """Specifically invoke p4 on the command supplied. """
120     real_cmd = p4_build_cmd(c)
121     return read_pipe_lines(real_cmd)
122
123 def p4_has_command(cmd):
124     """Ask p4 for help on this command.  If it returns an error, the
125        command does not exist in this version of p4."""
126     real_cmd = p4_build_cmd(["help", cmd])
127     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
128                                    stderr=subprocess.PIPE)
129     p.communicate()
130     return p.returncode == 0
131
132 def system(cmd):
133     expand = isinstance(cmd,basestring)
134     if verbose:
135         sys.stderr.write("executing %s\n" % str(cmd))
136     subprocess.check_call(cmd, shell=expand)
137
138 def p4_system(cmd):
139     """Specifically invoke p4 as the system command. """
140     real_cmd = p4_build_cmd(cmd)
141     expand = isinstance(real_cmd, basestring)
142     subprocess.check_call(real_cmd, shell=expand)
143
144 def p4_integrate(src, dest):
145     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
146
147 def p4_sync(f, *options):
148     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
149
150 def p4_add(f):
151     # forcibly add file names with wildcards
152     if wildcard_present(f):
153         p4_system(["add", "-f", f])
154     else:
155         p4_system(["add", f])
156
157 def p4_delete(f):
158     p4_system(["delete", wildcard_encode(f)])
159
160 def p4_edit(f):
161     p4_system(["edit", wildcard_encode(f)])
162
163 def p4_revert(f):
164     p4_system(["revert", wildcard_encode(f)])
165
166 def p4_reopen(type, f):
167     p4_system(["reopen", "-t", type, wildcard_encode(f)])
168
169 def p4_move(src, dest):
170     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
171
172 #
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers.  See "p4 help filetypes"
175 # for a list and explanation.
176 #
177 def split_p4_type(p4type):
178
179     p4_filetypes_historical = {
180         "ctempobj": "binary+Sw",
181         "ctext": "text+C",
182         "cxtext": "text+Cx",
183         "ktext": "text+k",
184         "kxtext": "text+kx",
185         "ltext": "text+F",
186         "tempobj": "binary+FSw",
187         "ubinary": "binary+F",
188         "uresource": "resource+F",
189         "uxbinary": "binary+Fx",
190         "xbinary": "binary+x",
191         "xltext": "text+Fx",
192         "xtempobj": "binary+Swx",
193         "xtext": "text+x",
194         "xunicode": "unicode+x",
195         "xutf16": "utf16+x",
196     }
197     if p4type in p4_filetypes_historical:
198         p4type = p4_filetypes_historical[p4type]
199     mods = ""
200     s = p4type.split("+")
201     base = s[0]
202     mods = ""
203     if len(s) > 1:
204         mods = s[1]
205     return (base, mods)
206
207 #
208 # return the raw p4 type of a file (text, text+ko, etc)
209 #
210 def p4_type(file):
211     results = p4CmdList(["fstat", "-T", "headType", file])
212     return results[0]['headType']
213
214 #
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
217 #
218 def p4_keywords_regexp_for_type(base, type_mods):
219     if base in ("text", "unicode", "binary"):
220         kwords = None
221         if "ko" in type_mods:
222             kwords = 'Id|Header'
223         elif "k" in type_mods:
224             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
225         else:
226             return None
227         pattern = r"""
228             \$              # Starts with a dollar, followed by...
229             (%s)            # one of the keywords, followed by...
230             (:[^$]+)?       # possibly an old expansion, followed by...
231             \$              # another dollar
232             """ % kwords
233         return pattern
234     else:
235         return None
236
237 #
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
241 #
242 def p4_keywords_regexp_for_file(file):
243     if not os.path.exists(file):
244         return None
245     else:
246         (type_base, type_mods) = split_p4_type(p4_type(file))
247         return p4_keywords_regexp_for_type(type_base, type_mods)
248
249 def setP4ExecBit(file, mode):
250     # Reopens an already open file and changes the execute bit to match
251     # the execute bit setting in the passed in mode.
252
253     p4Type = "+x"
254
255     if not isModeExec(mode):
256         p4Type = getP4OpenedType(file)
257         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
258         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
259         if p4Type[-1] == "+":
260             p4Type = p4Type[0:-1]
261
262     p4_reopen(p4Type, file)
263
264 def getP4OpenedType(file):
265     # Returns the perforce file type for the given file.
266
267     result = p4_read_pipe(["opened", wildcard_encode(file)])
268     match = re.match(".*\((.+)\)\r?$", result)
269     if match:
270         return match.group(1)
271     else:
272         die("Could not determine file type for %s (result: '%s')" % (file, result))
273
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths):
276     labels = set()
277     if isinstance(depotPaths,basestring):
278         depotPaths = [depotPaths]
279
280     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
281         label = l['label']
282         labels.add(label)
283
284     return labels
285
286 # Return the set of all git tags
287 def getGitTags():
288     gitTags = set()
289     for line in read_pipe_lines(["git", "tag"]):
290         tag = line.strip()
291         gitTags.add(tag)
292     return gitTags
293
294 def diffTreePattern():
295     # This is a simple generator for the diff tree regex pattern. This could be
296     # a class variable if this and parseDiffTreeEntry were a part of a class.
297     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
298     while True:
299         yield pattern
300
301 def parseDiffTreeEntry(entry):
302     """Parses a single diff tree entry into its component elements.
303
304     See git-diff-tree(1) manpage for details about the format of the diff
305     output. This method returns a dictionary with the following elements:
306
307     src_mode - The mode of the source file
308     dst_mode - The mode of the destination file
309     src_sha1 - The sha1 for the source file
310     dst_sha1 - The sha1 fr the destination file
311     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312     status_score - The score for the status (applicable for 'C' and 'R'
313                    statuses). This is None if there is no score.
314     src - The path for the source file.
315     dst - The path for the destination file. This is only present for
316           copy or renames. If it is not present, this is None.
317
318     If the pattern is not matched, None is returned."""
319
320     match = diffTreePattern().next().match(entry)
321     if match:
322         return {
323             'src_mode': match.group(1),
324             'dst_mode': match.group(2),
325             'src_sha1': match.group(3),
326             'dst_sha1': match.group(4),
327             'status': match.group(5),
328             'status_score': match.group(6),
329             'src': match.group(7),
330             'dst': match.group(10)
331         }
332     return None
333
334 def isModeExec(mode):
335     # Returns True if the given git mode represents an executable file,
336     # otherwise False.
337     return mode[-3:] == "755"
338
339 def isModeExecChanged(src_mode, dst_mode):
340     return isModeExec(src_mode) != isModeExec(dst_mode)
341
342 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
343
344     if isinstance(cmd,basestring):
345         cmd = "-G " + cmd
346         expand = True
347     else:
348         cmd = ["-G"] + cmd
349         expand = False
350
351     cmd = p4_build_cmd(cmd)
352     if verbose:
353         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
354
355     # Use a temporary file to avoid deadlocks without
356     # subprocess.communicate(), which would put another copy
357     # of stdout into memory.
358     stdin_file = None
359     if stdin is not None:
360         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
361         if isinstance(stdin,basestring):
362             stdin_file.write(stdin)
363         else:
364             for i in stdin:
365                 stdin_file.write(i + '\n')
366         stdin_file.flush()
367         stdin_file.seek(0)
368
369     p4 = subprocess.Popen(cmd,
370                           shell=expand,
371                           stdin=stdin_file,
372                           stdout=subprocess.PIPE)
373
374     result = []
375     try:
376         while True:
377             entry = marshal.load(p4.stdout)
378             if cb is not None:
379                 cb(entry)
380             else:
381                 result.append(entry)
382     except EOFError:
383         pass
384     exitCode = p4.wait()
385     if exitCode != 0:
386         entry = {}
387         entry["p4ExitCode"] = exitCode
388         result.append(entry)
389
390     return result
391
392 def p4Cmd(cmd):
393     list = p4CmdList(cmd)
394     result = {}
395     for entry in list:
396         result.update(entry)
397     return result;
398
399 def p4Where(depotPath):
400     if not depotPath.endswith("/"):
401         depotPath += "/"
402     depotPath = depotPath + "..."
403     outputList = p4CmdList(["where", depotPath])
404     output = None
405     for entry in outputList:
406         if "depotFile" in entry:
407             if entry["depotFile"] == depotPath:
408                 output = entry
409                 break
410         elif "data" in entry:
411             data = entry.get("data")
412             space = data.find(" ")
413             if data[:space] == depotPath:
414                 output = entry
415                 break
416     if output == None:
417         return ""
418     if output["code"] == "error":
419         return ""
420     clientPath = ""
421     if "path" in output:
422         clientPath = output.get("path")
423     elif "data" in output:
424         data = output.get("data")
425         lastSpace = data.rfind(" ")
426         clientPath = data[lastSpace + 1:]
427
428     if clientPath.endswith("..."):
429         clientPath = clientPath[:-3]
430     return clientPath
431
432 def currentGitBranch():
433     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
434
435 def isValidGitDir(path):
436     if (os.path.exists(path + "/HEAD")
437         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
438         return True;
439     return False
440
441 def parseRevision(ref):
442     return read_pipe("git rev-parse %s" % ref).strip()
443
444 def branchExists(ref):
445     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
446                      ignore_error=True)
447     return len(rev) > 0
448
449 def extractLogMessageFromGitCommit(commit):
450     logMessage = ""
451
452     ## fixme: title is first line of commit, not 1st paragraph.
453     foundTitle = False
454     for log in read_pipe_lines("git cat-file commit %s" % commit):
455        if not foundTitle:
456            if len(log) == 1:
457                foundTitle = True
458            continue
459
460        logMessage += log
461     return logMessage
462
463 def extractSettingsGitLog(log):
464     values = {}
465     for line in log.split("\n"):
466         line = line.strip()
467         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
468         if not m:
469             continue
470
471         assignments = m.group(1).split (':')
472         for a in assignments:
473             vals = a.split ('=')
474             key = vals[0].strip()
475             val = ('='.join (vals[1:])).strip()
476             if val.endswith ('\"') and val.startswith('"'):
477                 val = val[1:-1]
478
479             values[key] = val
480
481     paths = values.get("depot-paths")
482     if not paths:
483         paths = values.get("depot-path")
484     if paths:
485         values['depot-paths'] = paths.split(',')
486     return values
487
488 def gitBranchExists(branch):
489     proc = subprocess.Popen(["git", "rev-parse", branch],
490                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
491     return proc.wait() == 0;
492
493 _gitConfig = {}
494 def gitConfig(key, args = None): # set args to "--bool", for instance
495     if not _gitConfig.has_key(key):
496         argsFilter = ""
497         if args != None:
498             argsFilter = "%s " % args
499         cmd = "git config %s%s" % (argsFilter, key)
500         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
501     return _gitConfig[key]
502
503 def gitConfigList(key):
504     if not _gitConfig.has_key(key):
505         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
506     return _gitConfig[key]
507
508 def p4BranchesInGit(branchesAreInRemotes = True):
509     branches = {}
510
511     cmdline = "git rev-parse --symbolic "
512     if branchesAreInRemotes:
513         cmdline += " --remotes"
514     else:
515         cmdline += " --branches"
516
517     for line in read_pipe_lines(cmdline):
518         line = line.strip()
519
520         ## only import to p4/
521         if not line.startswith('p4/') or line == "p4/HEAD":
522             continue
523         branch = line
524
525         # strip off p4
526         branch = re.sub ("^p4/", "", line)
527
528         branches[branch] = parseRevision(line)
529     return branches
530
531 def findUpstreamBranchPoint(head = "HEAD"):
532     branches = p4BranchesInGit()
533     # map from depot-path to branch name
534     branchByDepotPath = {}
535     for branch in branches.keys():
536         tip = branches[branch]
537         log = extractLogMessageFromGitCommit(tip)
538         settings = extractSettingsGitLog(log)
539         if settings.has_key("depot-paths"):
540             paths = ",".join(settings["depot-paths"])
541             branchByDepotPath[paths] = "remotes/p4/" + branch
542
543     settings = None
544     parent = 0
545     while parent < 65535:
546         commit = head + "~%s" % parent
547         log = extractLogMessageFromGitCommit(commit)
548         settings = extractSettingsGitLog(log)
549         if settings.has_key("depot-paths"):
550             paths = ",".join(settings["depot-paths"])
551             if branchByDepotPath.has_key(paths):
552                 return [branchByDepotPath[paths], settings]
553
554         parent = parent + 1
555
556     return ["", settings]
557
558 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
559     if not silent:
560         print ("Creating/updating branch(es) in %s based on origin branch(es)"
561                % localRefPrefix)
562
563     originPrefix = "origin/p4/"
564
565     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
566         line = line.strip()
567         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
568             continue
569
570         headName = line[len(originPrefix):]
571         remoteHead = localRefPrefix + headName
572         originHead = line
573
574         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
575         if (not original.has_key('depot-paths')
576             or not original.has_key('change')):
577             continue
578
579         update = False
580         if not gitBranchExists(remoteHead):
581             if verbose:
582                 print "creating %s" % remoteHead
583             update = True
584         else:
585             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
586             if settings.has_key('change') > 0:
587                 if settings['depot-paths'] == original['depot-paths']:
588                     originP4Change = int(original['change'])
589                     p4Change = int(settings['change'])
590                     if originP4Change > p4Change:
591                         print ("%s (%s) is newer than %s (%s). "
592                                "Updating p4 branch from origin."
593                                % (originHead, originP4Change,
594                                   remoteHead, p4Change))
595                         update = True
596                 else:
597                     print ("Ignoring: %s was imported from %s while "
598                            "%s was imported from %s"
599                            % (originHead, ','.join(original['depot-paths']),
600                               remoteHead, ','.join(settings['depot-paths'])))
601
602         if update:
603             system("git update-ref %s %s" % (remoteHead, originHead))
604
605 def originP4BranchesExist():
606         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
607
608 def p4ChangesForPaths(depotPaths, changeRange):
609     assert depotPaths
610     cmd = ['changes']
611     for p in depotPaths:
612         cmd += ["%s...%s" % (p, changeRange)]
613     output = p4_read_pipe_lines(cmd)
614
615     changes = {}
616     for line in output:
617         changeNum = int(line.split(" ")[1])
618         changes[changeNum] = True
619
620     changelist = changes.keys()
621     changelist.sort()
622     return changelist
623
624 def p4PathStartsWith(path, prefix):
625     # This method tries to remedy a potential mixed-case issue:
626     #
627     # If UserA adds  //depot/DirA/file1
628     # and UserB adds //depot/dira/file2
629     #
630     # we may or may not have a problem. If you have core.ignorecase=true,
631     # we treat DirA and dira as the same directory
632     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
633     if ignorecase:
634         return path.lower().startswith(prefix.lower())
635     return path.startswith(prefix)
636
637 def getClientSpec():
638     """Look at the p4 client spec, create a View() object that contains
639        all the mappings, and return it."""
640
641     specList = p4CmdList("client -o")
642     if len(specList) != 1:
643         die('Output from "client -o" is %d lines, expecting 1' %
644             len(specList))
645
646     # dictionary of all client parameters
647     entry = specList[0]
648
649     # just the keys that start with "View"
650     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
651
652     # hold this new View
653     view = View()
654
655     # append the lines, in order, to the view
656     for view_num in range(len(view_keys)):
657         k = "View%d" % view_num
658         if k not in view_keys:
659             die("Expected view key %s missing" % k)
660         view.append(entry[k])
661
662     return view
663
664 def getClientRoot():
665     """Grab the client directory."""
666
667     output = p4CmdList("client -o")
668     if len(output) != 1:
669         die('Output from "client -o" is %d lines, expecting 1' % len(output))
670
671     entry = output[0]
672     if "Root" not in entry:
673         die('Client has no "Root"')
674
675     return entry["Root"]
676
677 #
678 # P4 wildcards are not allowed in filenames.  P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
681 #
682 def wildcard_decode(path):
683     # Search for and fix just these four characters.  Do % last so
684     # that fixing it does not inadvertently create new %-escapes.
685     # Cannot have * in a filename in windows; untested as to
686     # what p4 would do in such a case.
687     if not platform.system() == "Windows":
688         path = path.replace("%2A", "*")
689     path = path.replace("%23", "#") \
690                .replace("%40", "@") \
691                .replace("%25", "%")
692     return path
693
694 def wildcard_encode(path):
695     # do % first to avoid double-encoding the %s introduced here
696     path = path.replace("%", "%25") \
697                .replace("*", "%2A") \
698                .replace("#", "%23") \
699                .replace("@", "%40")
700     return path
701
702 def wildcard_present(path):
703     return path.translate(None, "*#@%") != path
704
705 class Command:
706     def __init__(self):
707         self.usage = "usage: %prog [options]"
708         self.needsGit = True
709         self.verbose = False
710
711 class P4UserMap:
712     def __init__(self):
713         self.userMapFromPerforceServer = False
714         self.myP4UserId = None
715
716     def p4UserId(self):
717         if self.myP4UserId:
718             return self.myP4UserId
719
720         results = p4CmdList("user -o")
721         for r in results:
722             if r.has_key('User'):
723                 self.myP4UserId = r['User']
724                 return r['User']
725         die("Could not find your p4 user id")
726
727     def p4UserIsMe(self, p4User):
728         # return True if the given p4 user is actually me
729         me = self.p4UserId()
730         if not p4User or p4User != me:
731             return False
732         else:
733             return True
734
735     def getUserCacheFilename(self):
736         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
737         return home + "/.gitp4-usercache.txt"
738
739     def getUserMapFromPerforceServer(self):
740         if self.userMapFromPerforceServer:
741             return
742         self.users = {}
743         self.emails = {}
744
745         for output in p4CmdList("users"):
746             if not output.has_key("User"):
747                 continue
748             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
749             self.emails[output["Email"]] = output["User"]
750
751
752         s = ''
753         for (key, val) in self.users.items():
754             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
755
756         open(self.getUserCacheFilename(), "wb").write(s)
757         self.userMapFromPerforceServer = True
758
759     def loadUserMapFromCache(self):
760         self.users = {}
761         self.userMapFromPerforceServer = False
762         try:
763             cache = open(self.getUserCacheFilename(), "rb")
764             lines = cache.readlines()
765             cache.close()
766             for line in lines:
767                 entry = line.strip().split("\t")
768                 self.users[entry[0]] = entry[1]
769         except IOError:
770             self.getUserMapFromPerforceServer()
771
772 class P4Debug(Command):
773     def __init__(self):
774         Command.__init__(self)
775         self.options = []
776         self.description = "A tool to debug the output of p4 -G."
777         self.needsGit = False
778
779     def run(self, args):
780         j = 0
781         for output in p4CmdList(args):
782             print 'Element: %d' % j
783             j += 1
784             print output
785         return True
786
787 class P4RollBack(Command):
788     def __init__(self):
789         Command.__init__(self)
790         self.options = [
791             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
792         ]
793         self.description = "A tool to debug the multi-branch import. Don't use :)"
794         self.rollbackLocalBranches = False
795
796     def run(self, args):
797         if len(args) != 1:
798             return False
799         maxChange = int(args[0])
800
801         if "p4ExitCode" in p4Cmd("changes -m 1"):
802             die("Problems executing p4");
803
804         if self.rollbackLocalBranches:
805             refPrefix = "refs/heads/"
806             lines = read_pipe_lines("git rev-parse --symbolic --branches")
807         else:
808             refPrefix = "refs/remotes/"
809             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
810
811         for line in lines:
812             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
813                 line = line.strip()
814                 ref = refPrefix + line
815                 log = extractLogMessageFromGitCommit(ref)
816                 settings = extractSettingsGitLog(log)
817
818                 depotPaths = settings['depot-paths']
819                 change = settings['change']
820
821                 changed = False
822
823                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
824                                                            for p in depotPaths]))) == 0:
825                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
826                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
827                     continue
828
829                 while change and int(change) > maxChange:
830                     changed = True
831                     if self.verbose:
832                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
833                     system("git update-ref %s \"%s^\"" % (ref, ref))
834                     log = extractLogMessageFromGitCommit(ref)
835                     settings =  extractSettingsGitLog(log)
836
837
838                     depotPaths = settings['depot-paths']
839                     change = settings['change']
840
841                 if changed:
842                     print "%s rewound to %s" % (ref, change)
843
844         return True
845
846 class P4Submit(Command, P4UserMap):
847
848     conflict_behavior_choices = ("ask", "skip", "quit")
849
850     def __init__(self):
851         Command.__init__(self)
852         P4UserMap.__init__(self)
853         self.options = [
854                 optparse.make_option("--origin", dest="origin"),
855                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
856                 # preserve the user, requires relevant p4 permissions
857                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
858                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
859                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
860                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
861                 optparse.make_option("--conflict", dest="conflict_behavior",
862                                      choices=self.conflict_behavior_choices)
863         ]
864         self.description = "Submit changes from git to the perforce depot."
865         self.usage += " [name of git branch to submit into perforce depot]"
866         self.origin = ""
867         self.detectRenames = False
868         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
869         self.dry_run = False
870         self.prepare_p4_only = False
871         self.conflict_behavior = None
872         self.isWindows = (platform.system() == "Windows")
873         self.exportLabels = False
874         self.p4HasMoveCommand = p4_has_command("move")
875
876     def check(self):
877         if len(p4CmdList("opened ...")) > 0:
878             die("You have files opened with perforce! Close them before starting the sync.")
879
880     def separate_jobs_from_description(self, message):
881         """Extract and return a possible Jobs field in the commit
882            message.  It goes into a separate section in the p4 change
883            specification.
884
885            A jobs line starts with "Jobs:" and looks like a new field
886            in a form.  Values are white-space separated on the same
887            line or on following lines that start with a tab.
888
889            This does not parse and extract the full git commit message
890            like a p4 form.  It just sees the Jobs: line as a marker
891            to pass everything from then on directly into the p4 form,
892            but outside the description section.
893
894            Return a tuple (stripped log message, jobs string)."""
895
896         m = re.search(r'^Jobs:', message, re.MULTILINE)
897         if m is None:
898             return (message, None)
899
900         jobtext = message[m.start():]
901         stripped_message = message[:m.start()].rstrip()
902         return (stripped_message, jobtext)
903
904     def prepareLogMessage(self, template, message, jobs):
905         """Edits the template returned from "p4 change -o" to insert
906            the message in the Description field, and the jobs text in
907            the Jobs field."""
908         result = ""
909
910         inDescriptionSection = False
911
912         for line in template.split("\n"):
913             if line.startswith("#"):
914                 result += line + "\n"
915                 continue
916
917             if inDescriptionSection:
918                 if line.startswith("Files:") or line.startswith("Jobs:"):
919                     inDescriptionSection = False
920                     # insert Jobs section
921                     if jobs:
922                         result += jobs + "\n"
923                 else:
924                     continue
925             else:
926                 if line.startswith("Description:"):
927                     inDescriptionSection = True
928                     line += "\n"
929                     for messageLine in message.split("\n"):
930                         line += "\t" + messageLine + "\n"
931
932             result += line + "\n"
933
934         return result
935
936     def patchRCSKeywords(self, file, pattern):
937         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
938         (handle, outFileName) = tempfile.mkstemp(dir='.')
939         try:
940             outFile = os.fdopen(handle, "w+")
941             inFile = open(file, "r")
942             regexp = re.compile(pattern, re.VERBOSE)
943             for line in inFile.readlines():
944                 line = regexp.sub(r'$\1$', line)
945                 outFile.write(line)
946             inFile.close()
947             outFile.close()
948             # Forcibly overwrite the original file
949             os.unlink(file)
950             shutil.move(outFileName, file)
951         except:
952             # cleanup our temporary file
953             os.unlink(outFileName)
954             print "Failed to strip RCS keywords in %s" % file
955             raise
956
957         print "Patched up RCS keywords in %s" % file
958
959     def p4UserForCommit(self,id):
960         # Return the tuple (perforce user,git email) for a given git commit id
961         self.getUserMapFromPerforceServer()
962         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
963         gitEmail = gitEmail.strip()
964         if not self.emails.has_key(gitEmail):
965             return (None,gitEmail)
966         else:
967             return (self.emails[gitEmail],gitEmail)
968
969     def checkValidP4Users(self,commits):
970         # check if any git authors cannot be mapped to p4 users
971         for id in commits:
972             (user,email) = self.p4UserForCommit(id)
973             if not user:
974                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
975                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
976                     print "%s" % msg
977                 else:
978                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
979
980     def lastP4Changelist(self):
981         # Get back the last changelist number submitted in this client spec. This
982         # then gets used to patch up the username in the change. If the same
983         # client spec is being used by multiple processes then this might go
984         # wrong.
985         results = p4CmdList("client -o")        # find the current client
986         client = None
987         for r in results:
988             if r.has_key('Client'):
989                 client = r['Client']
990                 break
991         if not client:
992             die("could not get client spec")
993         results = p4CmdList(["changes", "-c", client, "-m", "1"])
994         for r in results:
995             if r.has_key('change'):
996                 return r['change']
997         die("Could not get changelist number for last submit - cannot patch up user details")
998
999     def modifyChangelistUser(self, changelist, newUser):
1000         # fixup the user field of a changelist after it has been submitted.
1001         changes = p4CmdList("change -o %s" % changelist)
1002         if len(changes) != 1:
1003             die("Bad output from p4 change modifying %s to user %s" %
1004                 (changelist, newUser))
1005
1006         c = changes[0]
1007         if c['User'] == newUser: return   # nothing to do
1008         c['User'] = newUser
1009         input = marshal.dumps(c)
1010
1011         result = p4CmdList("change -f -i", stdin=input)
1012         for r in result:
1013             if r.has_key('code'):
1014                 if r['code'] == 'error':
1015                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1016             if r.has_key('data'):
1017                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1018                 return
1019         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1020
1021     def canChangeChangelists(self):
1022         # check to see if we have p4 admin or super-user permissions, either of
1023         # which are required to modify changelists.
1024         results = p4CmdList(["protects", self.depotPath])
1025         for r in results:
1026             if r.has_key('perm'):
1027                 if r['perm'] == 'admin':
1028                     return 1
1029                 if r['perm'] == 'super':
1030                     return 1
1031         return 0
1032
1033     def prepareSubmitTemplate(self):
1034         """Run "p4 change -o" to grab a change specification template.
1035            This does not use "p4 -G", as it is nice to keep the submission
1036            template in original order, since a human might edit it.
1037
1038            Remove lines in the Files section that show changes to files
1039            outside the depot path we're committing into."""
1040
1041         template = ""
1042         inFilesSection = False
1043         for line in p4_read_pipe_lines(['change', '-o']):
1044             if line.endswith("\r\n"):
1045                 line = line[:-2] + "\n"
1046             if inFilesSection:
1047                 if line.startswith("\t"):
1048                     # path starts and ends with a tab
1049                     path = line[1:]
1050                     lastTab = path.rfind("\t")
1051                     if lastTab != -1:
1052                         path = path[:lastTab]
1053                         if not p4PathStartsWith(path, self.depotPath):
1054                             continue
1055                 else:
1056                     inFilesSection = False
1057             else:
1058                 if line.startswith("Files:"):
1059                     inFilesSection = True
1060
1061             template += line
1062
1063         return template
1064
1065     def edit_template(self, template_file):
1066         """Invoke the editor to let the user change the submission
1067            message.  Return true if okay to continue with the submit."""
1068
1069         # if configured to skip the editing part, just submit
1070         if gitConfig("git-p4.skipSubmitEdit") == "true":
1071             return True
1072
1073         # look at the modification time, to check later if the user saved
1074         # the file
1075         mtime = os.stat(template_file).st_mtime
1076
1077         # invoke the editor
1078         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1079             editor = os.environ.get("P4EDITOR")
1080         else:
1081             editor = read_pipe("git var GIT_EDITOR").strip()
1082         system(editor + " " + template_file)
1083
1084         # If the file was not saved, prompt to see if this patch should
1085         # be skipped.  But skip this verification step if configured so.
1086         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1087             return True
1088
1089         # modification time updated means user saved the file
1090         if os.stat(template_file).st_mtime > mtime:
1091             return True
1092
1093         while True:
1094             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1095             if response == 'y':
1096                 return True
1097             if response == 'n':
1098                 return False
1099
1100     def applyCommit(self, id):
1101         """Apply one commit, return True if it succeeded."""
1102
1103         print "Applying", read_pipe(["git", "show", "-s",
1104                                      "--format=format:%h %s", id])
1105
1106         (p4User, gitEmail) = self.p4UserForCommit(id)
1107
1108         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1109         filesToAdd = set()
1110         filesToDelete = set()
1111         editedFiles = set()
1112         pureRenameCopy = set()
1113         filesToChangeExecBit = {}
1114
1115         for line in diff:
1116             diff = parseDiffTreeEntry(line)
1117             modifier = diff['status']
1118             path = diff['src']
1119             if modifier == "M":
1120                 p4_edit(path)
1121                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1122                     filesToChangeExecBit[path] = diff['dst_mode']
1123                 editedFiles.add(path)
1124             elif modifier == "A":
1125                 filesToAdd.add(path)
1126                 filesToChangeExecBit[path] = diff['dst_mode']
1127                 if path in filesToDelete:
1128                     filesToDelete.remove(path)
1129             elif modifier == "D":
1130                 filesToDelete.add(path)
1131                 if path in filesToAdd:
1132                     filesToAdd.remove(path)
1133             elif modifier == "C":
1134                 src, dest = diff['src'], diff['dst']
1135                 p4_integrate(src, dest)
1136                 pureRenameCopy.add(dest)
1137                 if diff['src_sha1'] != diff['dst_sha1']:
1138                     p4_edit(dest)
1139                     pureRenameCopy.discard(dest)
1140                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1141                     p4_edit(dest)
1142                     pureRenameCopy.discard(dest)
1143                     filesToChangeExecBit[dest] = diff['dst_mode']
1144                 os.unlink(dest)
1145                 editedFiles.add(dest)
1146             elif modifier == "R":
1147                 src, dest = diff['src'], diff['dst']
1148                 if self.p4HasMoveCommand:
1149                     p4_edit(src)        # src must be open before move
1150                     p4_move(src, dest)  # opens for (move/delete, move/add)
1151                 else:
1152                     p4_integrate(src, dest)
1153                     if diff['src_sha1'] != diff['dst_sha1']:
1154                         p4_edit(dest)
1155                     else:
1156                         pureRenameCopy.add(dest)
1157                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1158                     if not self.p4HasMoveCommand:
1159                         p4_edit(dest)   # with move: already open, writable
1160                     filesToChangeExecBit[dest] = diff['dst_mode']
1161                 if not self.p4HasMoveCommand:
1162                     os.unlink(dest)
1163                     filesToDelete.add(src)
1164                 editedFiles.add(dest)
1165             else:
1166                 die("unknown modifier %s for %s" % (modifier, path))
1167
1168         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1169         patchcmd = diffcmd + " | git apply "
1170         tryPatchCmd = patchcmd + "--check -"
1171         applyPatchCmd = patchcmd + "--check --apply -"
1172         patch_succeeded = True
1173
1174         if os.system(tryPatchCmd) != 0:
1175             fixed_rcs_keywords = False
1176             patch_succeeded = False
1177             print "Unfortunately applying the change failed!"
1178
1179             # Patch failed, maybe it's just RCS keyword woes. Look through
1180             # the patch to see if that's possible.
1181             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1182                 file = None
1183                 pattern = None
1184                 kwfiles = {}
1185                 for file in editedFiles | filesToDelete:
1186                     # did this file's delta contain RCS keywords?
1187                     pattern = p4_keywords_regexp_for_file(file)
1188
1189                     if pattern:
1190                         # this file is a possibility...look for RCS keywords.
1191                         regexp = re.compile(pattern, re.VERBOSE)
1192                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1193                             if regexp.search(line):
1194                                 if verbose:
1195                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1196                                 kwfiles[file] = pattern
1197                                 break
1198
1199                 for file in kwfiles:
1200                     if verbose:
1201                         print "zapping %s with %s" % (line,pattern)
1202                     self.patchRCSKeywords(file, kwfiles[file])
1203                     fixed_rcs_keywords = True
1204
1205             if fixed_rcs_keywords:
1206                 print "Retrying the patch with RCS keywords cleaned up"
1207                 if os.system(tryPatchCmd) == 0:
1208                     patch_succeeded = True
1209
1210         if not patch_succeeded:
1211             for f in editedFiles:
1212                 p4_revert(f)
1213             return False
1214
1215         #
1216         # Apply the patch for real, and do add/delete/+x handling.
1217         #
1218         system(applyPatchCmd)
1219
1220         for f in filesToAdd:
1221             p4_add(f)
1222         for f in filesToDelete:
1223             p4_revert(f)
1224             p4_delete(f)
1225
1226         # Set/clear executable bits
1227         for f in filesToChangeExecBit.keys():
1228             mode = filesToChangeExecBit[f]
1229             setP4ExecBit(f, mode)
1230
1231         #
1232         # Build p4 change description, starting with the contents
1233         # of the git commit message.
1234         #
1235         logMessage = extractLogMessageFromGitCommit(id)
1236         logMessage = logMessage.strip()
1237         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1238
1239         template = self.prepareSubmitTemplate()
1240         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1241
1242         if self.preserveUser:
1243            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1244
1245         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1246             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1247             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1248             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1249
1250         separatorLine = "######## everything below this line is just the diff #######\n"
1251
1252         # diff
1253         if os.environ.has_key("P4DIFF"):
1254             del(os.environ["P4DIFF"])
1255         diff = ""
1256         for editedFile in editedFiles:
1257             diff += p4_read_pipe(['diff', '-du',
1258                                   wildcard_encode(editedFile)])
1259
1260         # new file diff
1261         newdiff = ""
1262         for newFile in filesToAdd:
1263             newdiff += "==== new file ====\n"
1264             newdiff += "--- /dev/null\n"
1265             newdiff += "+++ %s\n" % newFile
1266             f = open(newFile, "r")
1267             for line in f.readlines():
1268                 newdiff += "+" + line
1269             f.close()
1270
1271         # change description file: submitTemplate, separatorLine, diff, newdiff
1272         (handle, fileName) = tempfile.mkstemp()
1273         tmpFile = os.fdopen(handle, "w+")
1274         if self.isWindows:
1275             submitTemplate = submitTemplate.replace("\n", "\r\n")
1276             separatorLine = separatorLine.replace("\n", "\r\n")
1277             newdiff = newdiff.replace("\n", "\r\n")
1278         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1279         tmpFile.close()
1280
1281         if self.prepare_p4_only:
1282             #
1283             # Leave the p4 tree prepared, and the submit template around
1284             # and let the user decide what to do next
1285             #
1286             print
1287             print "P4 workspace prepared for submission."
1288             print "To submit or revert, go to client workspace"
1289             print "  " + self.clientPath
1290             print
1291             print "To submit, use \"p4 submit\" to write a new description,"
1292             print "or \"p4 submit -i %s\" to use the one prepared by" \
1293                   " \"git p4\"." % fileName
1294             print "You can delete the file \"%s\" when finished." % fileName
1295
1296             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1297                 print "To preserve change ownership by user %s, you must\n" \
1298                       "do \"p4 change -f <change>\" after submitting and\n" \
1299                       "edit the User field."
1300             if pureRenameCopy:
1301                 print "After submitting, renamed files must be re-synced."
1302                 print "Invoke \"p4 sync -f\" on each of these files:"
1303                 for f in pureRenameCopy:
1304                     print "  " + f
1305
1306             print
1307             print "To revert the changes, use \"p4 revert ...\", and delete"
1308             print "the submit template file \"%s\"" % fileName
1309             if filesToAdd:
1310                 print "Since the commit adds new files, they must be deleted:"
1311                 for f in filesToAdd:
1312                     print "  " + f
1313             print
1314             return True
1315
1316         #
1317         # Let the user edit the change description, then submit it.
1318         #
1319         if self.edit_template(fileName):
1320             # read the edited message and submit
1321             ret = True
1322             tmpFile = open(fileName, "rb")
1323             message = tmpFile.read()
1324             tmpFile.close()
1325             submitTemplate = message[:message.index(separatorLine)]
1326             if self.isWindows:
1327                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1328             p4_write_pipe(['submit', '-i'], submitTemplate)
1329
1330             if self.preserveUser:
1331                 if p4User:
1332                     # Get last changelist number. Cannot easily get it from
1333                     # the submit command output as the output is
1334                     # unmarshalled.
1335                     changelist = self.lastP4Changelist()
1336                     self.modifyChangelistUser(changelist, p4User)
1337
1338             # The rename/copy happened by applying a patch that created a
1339             # new file.  This leaves it writable, which confuses p4.
1340             for f in pureRenameCopy:
1341                 p4_sync(f, "-f")
1342
1343         else:
1344             # skip this patch
1345             ret = False
1346             print "Submission cancelled, undoing p4 changes."
1347             for f in editedFiles:
1348                 p4_revert(f)
1349             for f in filesToAdd:
1350                 p4_revert(f)
1351                 os.remove(f)
1352             for f in filesToDelete:
1353                 p4_revert(f)
1354
1355         os.remove(fileName)
1356         return ret
1357
1358     # Export git tags as p4 labels. Create a p4 label and then tag
1359     # with that.
1360     def exportGitTags(self, gitTags):
1361         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1362         if len(validLabelRegexp) == 0:
1363             validLabelRegexp = defaultLabelRegexp
1364         m = re.compile(validLabelRegexp)
1365
1366         for name in gitTags:
1367
1368             if not m.match(name):
1369                 if verbose:
1370                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1371                 continue
1372
1373             # Get the p4 commit this corresponds to
1374             logMessage = extractLogMessageFromGitCommit(name)
1375             values = extractSettingsGitLog(logMessage)
1376
1377             if not values.has_key('change'):
1378                 # a tag pointing to something not sent to p4; ignore
1379                 if verbose:
1380                     print "git tag %s does not give a p4 commit" % name
1381                 continue
1382             else:
1383                 changelist = values['change']
1384
1385             # Get the tag details.
1386             inHeader = True
1387             isAnnotated = False
1388             body = []
1389             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1390                 l = l.strip()
1391                 if inHeader:
1392                     if re.match(r'tag\s+', l):
1393                         isAnnotated = True
1394                     elif re.match(r'\s*$', l):
1395                         inHeader = False
1396                         continue
1397                 else:
1398                     body.append(l)
1399
1400             if not isAnnotated:
1401                 body = ["lightweight tag imported by git p4\n"]
1402
1403             # Create the label - use the same view as the client spec we are using
1404             clientSpec = getClientSpec()
1405
1406             labelTemplate  = "Label: %s\n" % name
1407             labelTemplate += "Description:\n"
1408             for b in body:
1409                 labelTemplate += "\t" + b + "\n"
1410             labelTemplate += "View:\n"
1411             for mapping in clientSpec.mappings:
1412                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1413
1414             if self.dry_run:
1415                 print "Would create p4 label %s for tag" % name
1416             elif self.prepare_p4_only:
1417                 print "Not creating p4 label %s for tag due to option" \
1418                       " --prepare-p4-only" % name
1419             else:
1420                 p4_write_pipe(["label", "-i"], labelTemplate)
1421
1422                 # Use the label
1423                 p4_system(["tag", "-l", name] +
1424                           ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1425
1426                 if verbose:
1427                     print "created p4 label for tag %s" % name
1428
1429     def run(self, args):
1430         if len(args) == 0:
1431             self.master = currentGitBranch()
1432             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1433                 die("Detecting current git branch failed!")
1434         elif len(args) == 1:
1435             self.master = args[0]
1436             if not branchExists(self.master):
1437                 die("Branch %s does not exist" % self.master)
1438         else:
1439             return False
1440
1441         allowSubmit = gitConfig("git-p4.allowSubmit")
1442         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1443             die("%s is not in git-p4.allowSubmit" % self.master)
1444
1445         [upstream, settings] = findUpstreamBranchPoint()
1446         self.depotPath = settings['depot-paths'][0]
1447         if len(self.origin) == 0:
1448             self.origin = upstream
1449
1450         if self.preserveUser:
1451             if not self.canChangeChangelists():
1452                 die("Cannot preserve user names without p4 super-user or admin permissions")
1453
1454         # if not set from the command line, try the config file
1455         if self.conflict_behavior is None:
1456             val = gitConfig("git-p4.conflict")
1457             if val:
1458                 if val not in self.conflict_behavior_choices:
1459                     die("Invalid value '%s' for config git-p4.conflict" % val)
1460             else:
1461                 val = "ask"
1462             self.conflict_behavior = val
1463
1464         if self.verbose:
1465             print "Origin branch is " + self.origin
1466
1467         if len(self.depotPath) == 0:
1468             print "Internal error: cannot locate perforce depot path from existing branches"
1469             sys.exit(128)
1470
1471         self.useClientSpec = False
1472         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1473             self.useClientSpec = True
1474         if self.useClientSpec:
1475             self.clientSpecDirs = getClientSpec()
1476
1477         if self.useClientSpec:
1478             # all files are relative to the client spec
1479             self.clientPath = getClientRoot()
1480         else:
1481             self.clientPath = p4Where(self.depotPath)
1482
1483         if self.clientPath == "":
1484             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1485
1486         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1487         self.oldWorkingDirectory = os.getcwd()
1488
1489         # ensure the clientPath exists
1490         new_client_dir = False
1491         if not os.path.exists(self.clientPath):
1492             new_client_dir = True
1493             os.makedirs(self.clientPath)
1494
1495         chdir(self.clientPath)
1496         if self.dry_run:
1497             print "Would synchronize p4 checkout in %s" % self.clientPath
1498         else:
1499             print "Synchronizing p4 checkout..."
1500             if new_client_dir:
1501                 # old one was destroyed, and maybe nobody told p4
1502                 p4_sync("...", "-f")
1503             else:
1504                 p4_sync("...")
1505         self.check()
1506
1507         commits = []
1508         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1509             commits.append(line.strip())
1510         commits.reverse()
1511
1512         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1513             self.checkAuthorship = False
1514         else:
1515             self.checkAuthorship = True
1516
1517         if self.preserveUser:
1518             self.checkValidP4Users(commits)
1519
1520         #
1521         # Build up a set of options to be passed to diff when
1522         # submitting each commit to p4.
1523         #
1524         if self.detectRenames:
1525             # command-line -M arg
1526             self.diffOpts = "-M"
1527         else:
1528             # If not explicitly set check the config variable
1529             detectRenames = gitConfig("git-p4.detectRenames")
1530
1531             if detectRenames.lower() == "false" or detectRenames == "":
1532                 self.diffOpts = ""
1533             elif detectRenames.lower() == "true":
1534                 self.diffOpts = "-M"
1535             else:
1536                 self.diffOpts = "-M%s" % detectRenames
1537
1538         # no command-line arg for -C or --find-copies-harder, just
1539         # config variables
1540         detectCopies = gitConfig("git-p4.detectCopies")
1541         if detectCopies.lower() == "false" or detectCopies == "":
1542             pass
1543         elif detectCopies.lower() == "true":
1544             self.diffOpts += " -C"
1545         else:
1546             self.diffOpts += " -C%s" % detectCopies
1547
1548         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1549             self.diffOpts += " --find-copies-harder"
1550
1551         #
1552         # Apply the commits, one at a time.  On failure, ask if should
1553         # continue to try the rest of the patches, or quit.
1554         #
1555         if self.dry_run:
1556             print "Would apply"
1557         applied = []
1558         last = len(commits) - 1
1559         for i, commit in enumerate(commits):
1560             if self.dry_run:
1561                 print " ", read_pipe(["git", "show", "-s",
1562                                       "--format=format:%h %s", commit])
1563                 ok = True
1564             else:
1565                 ok = self.applyCommit(commit)
1566             if ok:
1567                 applied.append(commit)
1568             else:
1569                 if self.prepare_p4_only and i < last:
1570                     print "Processing only the first commit due to option" \
1571                           " --prepare-p4-only"
1572                     break
1573                 if i < last:
1574                     quit = False
1575                     while True:
1576                         # prompt for what to do, or use the option/variable
1577                         if self.conflict_behavior == "ask":
1578                             print "What do you want to do?"
1579                             response = raw_input("[s]kip this commit but apply"
1580                                                  " the rest, or [q]uit? ")
1581                             if not response:
1582                                 continue
1583                         elif self.conflict_behavior == "skip":
1584                             response = "s"
1585                         elif self.conflict_behavior == "quit":
1586                             response = "q"
1587                         else:
1588                             die("Unknown conflict_behavior '%s'" %
1589                                 self.conflict_behavior)
1590
1591                         if response[0] == "s":
1592                             print "Skipping this commit, but applying the rest"
1593                             break
1594                         if response[0] == "q":
1595                             print "Quitting"
1596                             quit = True
1597                             break
1598                     if quit:
1599                         break
1600
1601         chdir(self.oldWorkingDirectory)
1602
1603         if self.dry_run:
1604             pass
1605         elif self.prepare_p4_only:
1606             pass
1607         elif len(commits) == len(applied):
1608             print "All commits applied!"
1609
1610             sync = P4Sync()
1611             sync.run([])
1612
1613             rebase = P4Rebase()
1614             rebase.rebase()
1615
1616         else:
1617             if len(applied) == 0:
1618                 print "No commits applied."
1619             else:
1620                 print "Applied only the commits marked with '*':"
1621                 for c in commits:
1622                     if c in applied:
1623                         star = "*"
1624                     else:
1625                         star = " "
1626                     print star, read_pipe(["git", "show", "-s",
1627                                            "--format=format:%h %s",  c])
1628                 print "You will have to do 'git p4 sync' and rebase."
1629
1630         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1631             self.exportLabels = True
1632
1633         if self.exportLabels:
1634             p4Labels = getP4Labels(self.depotPath)
1635             gitTags = getGitTags()
1636
1637             missingGitTags = gitTags - p4Labels
1638             self.exportGitTags(missingGitTags)
1639
1640         # exit with error unless everything applied perfecly
1641         if len(commits) != len(applied):
1642                 sys.exit(1)
1643
1644         return True
1645
1646 class View(object):
1647     """Represent a p4 view ("p4 help views"), and map files in a
1648        repo according to the view."""
1649
1650     class Path(object):
1651         """A depot or client path, possibly containing wildcards.
1652            The only one supported is ... at the end, currently.
1653            Initialize with the full path, with //depot or //client."""
1654
1655         def __init__(self, path, is_depot):
1656             self.path = path
1657             self.is_depot = is_depot
1658             self.find_wildcards()
1659             # remember the prefix bit, useful for relative mappings
1660             m = re.match("(//[^/]+/)", self.path)
1661             if not m:
1662                 die("Path %s does not start with //prefix/" % self.path)
1663             prefix = m.group(1)
1664             if not self.is_depot:
1665                 # strip //client/ on client paths
1666                 self.path = self.path[len(prefix):]
1667
1668         def find_wildcards(self):
1669             """Make sure wildcards are valid, and set up internal
1670                variables."""
1671
1672             self.ends_triple_dot = False
1673             # There are three wildcards allowed in p4 views
1674             # (see "p4 help views").  This code knows how to
1675             # handle "..." (only at the end), but cannot deal with
1676             # "%%n" or "*".  Only check the depot_side, as p4 should
1677             # validate that the client_side matches too.
1678             if re.search(r'%%[1-9]', self.path):
1679                 die("Can't handle %%n wildcards in view: %s" % self.path)
1680             if self.path.find("*") >= 0:
1681                 die("Can't handle * wildcards in view: %s" % self.path)
1682             triple_dot_index = self.path.find("...")
1683             if triple_dot_index >= 0:
1684                 if triple_dot_index != len(self.path) - 3:
1685                     die("Can handle only single ... wildcard, at end: %s" %
1686                         self.path)
1687                 self.ends_triple_dot = True
1688
1689         def ensure_compatible(self, other_path):
1690             """Make sure the wildcards agree."""
1691             if self.ends_triple_dot != other_path.ends_triple_dot:
1692                  die("Both paths must end with ... if either does;\n" +
1693                      "paths: %s %s" % (self.path, other_path.path))
1694
1695         def match_wildcards(self, test_path):
1696             """See if this test_path matches us, and fill in the value
1697                of the wildcards if so.  Returns a tuple of
1698                (True|False, wildcards[]).  For now, only the ... at end
1699                is supported, so at most one wildcard."""
1700             if self.ends_triple_dot:
1701                 dotless = self.path[:-3]
1702                 if test_path.startswith(dotless):
1703                     wildcard = test_path[len(dotless):]
1704                     return (True, [ wildcard ])
1705             else:
1706                 if test_path == self.path:
1707                     return (True, [])
1708             return (False, [])
1709
1710         def match(self, test_path):
1711             """Just return if it matches; don't bother with the wildcards."""
1712             b, _ = self.match_wildcards(test_path)
1713             return b
1714
1715         def fill_in_wildcards(self, wildcards):
1716             """Return the relative path, with the wildcards filled in
1717                if there are any."""
1718             if self.ends_triple_dot:
1719                 return self.path[:-3] + wildcards[0]
1720             else:
1721                 return self.path
1722
1723     class Mapping(object):
1724         def __init__(self, depot_side, client_side, overlay, exclude):
1725             # depot_side is without the trailing /... if it had one
1726             self.depot_side = View.Path(depot_side, is_depot=True)
1727             self.client_side = View.Path(client_side, is_depot=False)
1728             self.overlay = overlay  # started with "+"
1729             self.exclude = exclude  # started with "-"
1730             assert not (self.overlay and self.exclude)
1731             self.depot_side.ensure_compatible(self.client_side)
1732
1733         def __str__(self):
1734             c = " "
1735             if self.overlay:
1736                 c = "+"
1737             if self.exclude:
1738                 c = "-"
1739             return "View.Mapping: %s%s -> %s" % \
1740                    (c, self.depot_side.path, self.client_side.path)
1741
1742         def map_depot_to_client(self, depot_path):
1743             """Calculate the client path if using this mapping on the
1744                given depot path; does not consider the effect of other
1745                mappings in a view.  Even excluded mappings are returned."""
1746             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1747             if not matches:
1748                 return ""
1749             client_path = self.client_side.fill_in_wildcards(wildcards)
1750             return client_path
1751
1752     #
1753     # View methods
1754     #
1755     def __init__(self):
1756         self.mappings = []
1757
1758     def append(self, view_line):
1759         """Parse a view line, splitting it into depot and client
1760            sides.  Append to self.mappings, preserving order."""
1761
1762         # Split the view line into exactly two words.  P4 enforces
1763         # structure on these lines that simplifies this quite a bit.
1764         #
1765         # Either or both words may be double-quoted.
1766         # Single quotes do not matter.
1767         # Double-quote marks cannot occur inside the words.
1768         # A + or - prefix is also inside the quotes.
1769         # There are no quotes unless they contain a space.
1770         # The line is already white-space stripped.
1771         # The two words are separated by a single space.
1772         #
1773         if view_line[0] == '"':
1774             # First word is double quoted.  Find its end.
1775             close_quote_index = view_line.find('"', 1)
1776             if close_quote_index <= 0:
1777                 die("No first-word closing quote found: %s" % view_line)
1778             depot_side = view_line[1:close_quote_index]
1779             # skip closing quote and space
1780             rhs_index = close_quote_index + 1 + 1
1781         else:
1782             space_index = view_line.find(" ")
1783             if space_index <= 0:
1784                 die("No word-splitting space found: %s" % view_line)
1785             depot_side = view_line[0:space_index]
1786             rhs_index = space_index + 1
1787
1788         if view_line[rhs_index] == '"':
1789             # Second word is double quoted.  Make sure there is a
1790             # double quote at the end too.
1791             if not view_line.endswith('"'):
1792                 die("View line with rhs quote should end with one: %s" %
1793                     view_line)
1794             # skip the quotes
1795             client_side = view_line[rhs_index+1:-1]
1796         else:
1797             client_side = view_line[rhs_index:]
1798
1799         # prefix + means overlay on previous mapping
1800         overlay = False
1801         if depot_side.startswith("+"):
1802             overlay = True
1803             depot_side = depot_side[1:]
1804
1805         # prefix - means exclude this path
1806         exclude = False
1807         if depot_side.startswith("-"):
1808             exclude = True
1809             depot_side = depot_side[1:]
1810
1811         m = View.Mapping(depot_side, client_side, overlay, exclude)
1812         self.mappings.append(m)
1813
1814     def map_in_client(self, depot_path):
1815         """Return the relative location in the client where this
1816            depot file should live.  Returns "" if the file should
1817            not be mapped in the client."""
1818
1819         paths_filled = []
1820         client_path = ""
1821
1822         # look at later entries first
1823         for m in self.mappings[::-1]:
1824
1825             # see where will this path end up in the client
1826             p = m.map_depot_to_client(depot_path)
1827
1828             if p == "":
1829                 # Depot path does not belong in client.  Must remember
1830                 # this, as previous items should not cause files to
1831                 # exist in this path either.  Remember that the list is
1832                 # being walked from the end, which has higher precedence.
1833                 # Overlap mappings do not exclude previous mappings.
1834                 if not m.overlay:
1835                     paths_filled.append(m.client_side)
1836
1837             else:
1838                 # This mapping matched; no need to search any further.
1839                 # But, the mapping could be rejected if the client path
1840                 # has already been claimed by an earlier mapping (i.e.
1841                 # one later in the list, which we are walking backwards).
1842                 already_mapped_in_client = False
1843                 for f in paths_filled:
1844                     # this is View.Path.match
1845                     if f.match(p):
1846                         already_mapped_in_client = True
1847                         break
1848                 if not already_mapped_in_client:
1849                     # Include this file, unless it is from a line that
1850                     # explicitly said to exclude it.
1851                     if not m.exclude:
1852                         client_path = p
1853
1854                 # a match, even if rejected, always stops the search
1855                 break
1856
1857         return client_path
1858
1859 class P4Sync(Command, P4UserMap):
1860     delete_actions = ( "delete", "move/delete", "purge" )
1861
1862     def __init__(self):
1863         Command.__init__(self)
1864         P4UserMap.__init__(self)
1865         self.options = [
1866                 optparse.make_option("--branch", dest="branch"),
1867                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1868                 optparse.make_option("--changesfile", dest="changesFile"),
1869                 optparse.make_option("--silent", dest="silent", action="store_true"),
1870                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1871                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1872                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1873                                      help="Import into refs/heads/ , not refs/remotes"),
1874                 optparse.make_option("--max-changes", dest="maxChanges"),
1875                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1876                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1877                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1878                                      help="Only sync files that are included in the Perforce Client Spec")
1879         ]
1880         self.description = """Imports from Perforce into a git repository.\n
1881     example:
1882     //depot/my/project/ -- to import the current head
1883     //depot/my/project/@all -- to import everything
1884     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1885
1886     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1887
1888         self.usage += " //depot/path[@revRange]"
1889         self.silent = False
1890         self.createdBranches = set()
1891         self.committedChanges = set()
1892         self.branch = ""
1893         self.detectBranches = False
1894         self.detectLabels = False
1895         self.importLabels = False
1896         self.changesFile = ""
1897         self.syncWithOrigin = True
1898         self.importIntoRemotes = True
1899         self.maxChanges = ""
1900         self.isWindows = (platform.system() == "Windows")
1901         self.keepRepoPath = False
1902         self.depotPaths = None
1903         self.p4BranchesInGit = []
1904         self.cloneExclude = []
1905         self.useClientSpec = False
1906         self.useClientSpec_from_options = False
1907         self.clientSpecDirs = None
1908         self.tempBranches = []
1909         self.tempBranchLocation = "git-p4-tmp"
1910
1911         if gitConfig("git-p4.syncFromOrigin") == "false":
1912             self.syncWithOrigin = False
1913
1914     # Force a checkpoint in fast-import and wait for it to finish
1915     def checkpoint(self):
1916         self.gitStream.write("checkpoint\n\n")
1917         self.gitStream.write("progress checkpoint\n\n")
1918         out = self.gitOutput.readline()
1919         if self.verbose:
1920             print "checkpoint finished: " + out
1921
1922     def extractFilesFromCommit(self, commit):
1923         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1924                              for path in self.cloneExclude]
1925         files = []
1926         fnum = 0
1927         while commit.has_key("depotFile%s" % fnum):
1928             path =  commit["depotFile%s" % fnum]
1929
1930             if [p for p in self.cloneExclude
1931                 if p4PathStartsWith(path, p)]:
1932                 found = False
1933             else:
1934                 found = [p for p in self.depotPaths
1935                          if p4PathStartsWith(path, p)]
1936             if not found:
1937                 fnum = fnum + 1
1938                 continue
1939
1940             file = {}
1941             file["path"] = path
1942             file["rev"] = commit["rev%s" % fnum]
1943             file["action"] = commit["action%s" % fnum]
1944             file["type"] = commit["type%s" % fnum]
1945             files.append(file)
1946             fnum = fnum + 1
1947         return files
1948
1949     def stripRepoPath(self, path, prefixes):
1950         """When streaming files, this is called to map a p4 depot path
1951            to where it should go in git.  The prefixes are either
1952            self.depotPaths, or self.branchPrefixes in the case of
1953            branch detection."""
1954
1955         if self.useClientSpec:
1956             # branch detection moves files up a level (the branch name)
1957             # from what client spec interpretation gives
1958             path = self.clientSpecDirs.map_in_client(path)
1959             if self.detectBranches:
1960                 for b in self.knownBranches:
1961                     if path.startswith(b + "/"):
1962                         path = path[len(b)+1:]
1963
1964         elif self.keepRepoPath:
1965             # Preserve everything in relative path name except leading
1966             # //depot/; just look at first prefix as they all should
1967             # be in the same depot.
1968             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
1969             if p4PathStartsWith(path, depot):
1970                 path = path[len(depot):]
1971
1972         else:
1973             for p in prefixes:
1974                 if p4PathStartsWith(path, p):
1975                     path = path[len(p):]
1976                     break
1977
1978         path = wildcard_decode(path)
1979         return path
1980
1981     def splitFilesIntoBranches(self, commit):
1982         """Look at each depotFile in the commit to figure out to what
1983            branch it belongs."""
1984
1985         branches = {}
1986         fnum = 0
1987         while commit.has_key("depotFile%s" % fnum):
1988             path =  commit["depotFile%s" % fnum]
1989             found = [p for p in self.depotPaths
1990                      if p4PathStartsWith(path, p)]
1991             if not found:
1992                 fnum = fnum + 1
1993                 continue
1994
1995             file = {}
1996             file["path"] = path
1997             file["rev"] = commit["rev%s" % fnum]
1998             file["action"] = commit["action%s" % fnum]
1999             file["type"] = commit["type%s" % fnum]
2000             fnum = fnum + 1
2001
2002             # start with the full relative path where this file would
2003             # go in a p4 client
2004             if self.useClientSpec:
2005                 relPath = self.clientSpecDirs.map_in_client(path)
2006             else:
2007                 relPath = self.stripRepoPath(path, self.depotPaths)
2008
2009             for branch in self.knownBranches.keys():
2010                 # add a trailing slash so that a commit into qt/4.2foo
2011                 # doesn't end up in qt/4.2, e.g.
2012                 if relPath.startswith(branch + "/"):
2013                     if branch not in branches:
2014                         branches[branch] = []
2015                     branches[branch].append(file)
2016                     break
2017
2018         return branches
2019
2020     # output one file from the P4 stream
2021     # - helper for streamP4Files
2022
2023     def streamOneP4File(self, file, contents):
2024         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2025         if verbose:
2026             sys.stderr.write("%s\n" % relPath)
2027
2028         (type_base, type_mods) = split_p4_type(file["type"])
2029
2030         git_mode = "100644"
2031         if "x" in type_mods:
2032             git_mode = "100755"
2033         if type_base == "symlink":
2034             git_mode = "120000"
2035             # p4 print on a symlink contains "target\n"; remove the newline
2036             data = ''.join(contents)
2037             contents = [data[:-1]]
2038
2039         if type_base == "utf16":
2040             # p4 delivers different text in the python output to -G
2041             # than it does when using "print -o", or normal p4 client
2042             # operations.  utf16 is converted to ascii or utf8, perhaps.
2043             # But ascii text saved as -t utf16 is completely mangled.
2044             # Invoke print -o to get the real contents.
2045             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2046             contents = [ text ]
2047
2048         if type_base == "apple":
2049             # Apple filetype files will be streamed as a concatenation of
2050             # its appledouble header and the contents.  This is useless
2051             # on both macs and non-macs.  If using "print -q -o xx", it
2052             # will create "xx" with the data, and "%xx" with the header.
2053             # This is also not very useful.
2054             #
2055             # Ideally, someday, this script can learn how to generate
2056             # appledouble files directly and import those to git, but
2057             # non-mac machines can never find a use for apple filetype.
2058             print "\nIgnoring apple filetype file %s" % file['depotFile']
2059             return
2060
2061         # Perhaps windows wants unicode, utf16 newlines translated too;
2062         # but this is not doing it.
2063         if self.isWindows and type_base == "text":
2064             mangled = []
2065             for data in contents:
2066                 data = data.replace("\r\n", "\n")
2067                 mangled.append(data)
2068             contents = mangled
2069
2070         # Note that we do not try to de-mangle keywords on utf16 files,
2071         # even though in theory somebody may want that.
2072         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2073         if pattern:
2074             regexp = re.compile(pattern, re.VERBOSE)
2075             text = ''.join(contents)
2076             text = regexp.sub(r'$\1$', text)
2077             contents = [ text ]
2078
2079         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2080
2081         # total length...
2082         length = 0
2083         for d in contents:
2084             length = length + len(d)
2085
2086         self.gitStream.write("data %d\n" % length)
2087         for d in contents:
2088             self.gitStream.write(d)
2089         self.gitStream.write("\n")
2090
2091     def streamOneP4Deletion(self, file):
2092         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2093         if verbose:
2094             sys.stderr.write("delete %s\n" % relPath)
2095         self.gitStream.write("D %s\n" % relPath)
2096
2097     # handle another chunk of streaming data
2098     def streamP4FilesCb(self, marshalled):
2099
2100         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2101             # start of a new file - output the old one first
2102             self.streamOneP4File(self.stream_file, self.stream_contents)
2103             self.stream_file = {}
2104             self.stream_contents = []
2105             self.stream_have_file_info = False
2106
2107         # pick up the new file information... for the
2108         # 'data' field we need to append to our array
2109         for k in marshalled.keys():
2110             if k == 'data':
2111                 self.stream_contents.append(marshalled['data'])
2112             else:
2113                 self.stream_file[k] = marshalled[k]
2114
2115         self.stream_have_file_info = True
2116
2117     # Stream directly from "p4 files" into "git fast-import"
2118     def streamP4Files(self, files):
2119         filesForCommit = []
2120         filesToRead = []
2121         filesToDelete = []
2122
2123         for f in files:
2124             # if using a client spec, only add the files that have
2125             # a path in the client
2126             if self.clientSpecDirs:
2127                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2128                     continue
2129
2130             filesForCommit.append(f)
2131             if f['action'] in self.delete_actions:
2132                 filesToDelete.append(f)
2133             else:
2134                 filesToRead.append(f)
2135
2136         # deleted files...
2137         for f in filesToDelete:
2138             self.streamOneP4Deletion(f)
2139
2140         if len(filesToRead) > 0:
2141             self.stream_file = {}
2142             self.stream_contents = []
2143             self.stream_have_file_info = False
2144
2145             # curry self argument
2146             def streamP4FilesCbSelf(entry):
2147                 self.streamP4FilesCb(entry)
2148
2149             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2150
2151             p4CmdList(["-x", "-", "print"],
2152                       stdin=fileArgs,
2153                       cb=streamP4FilesCbSelf)
2154
2155             # do the last chunk
2156             if self.stream_file.has_key('depotFile'):
2157                 self.streamOneP4File(self.stream_file, self.stream_contents)
2158
2159     def make_email(self, userid):
2160         if userid in self.users:
2161             return self.users[userid]
2162         else:
2163             return "%s <a@b>" % userid
2164
2165     # Stream a p4 tag
2166     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2167         if verbose:
2168             print "writing tag %s for commit %s" % (labelName, commit)
2169         gitStream.write("tag %s\n" % labelName)
2170         gitStream.write("from %s\n" % commit)
2171
2172         if labelDetails.has_key('Owner'):
2173             owner = labelDetails["Owner"]
2174         else:
2175             owner = None
2176
2177         # Try to use the owner of the p4 label, or failing that,
2178         # the current p4 user id.
2179         if owner:
2180             email = self.make_email(owner)
2181         else:
2182             email = self.make_email(self.p4UserId())
2183         tagger = "%s %s %s" % (email, epoch, self.tz)
2184
2185         gitStream.write("tagger %s\n" % tagger)
2186
2187         print "labelDetails=",labelDetails
2188         if labelDetails.has_key('Description'):
2189             description = labelDetails['Description']
2190         else:
2191             description = 'Label from git p4'
2192
2193         gitStream.write("data %d\n" % len(description))
2194         gitStream.write(description)
2195         gitStream.write("\n")
2196
2197     def commit(self, details, files, branch, parent = ""):
2198         epoch = details["time"]
2199         author = details["user"]
2200
2201         if self.verbose:
2202             print "commit into %s" % branch
2203
2204         # start with reading files; if that fails, we should not
2205         # create a commit.
2206         new_files = []
2207         for f in files:
2208             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2209                 new_files.append (f)
2210             else:
2211                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2212
2213         self.gitStream.write("commit %s\n" % branch)
2214 #        gitStream.write("mark :%s\n" % details["change"])
2215         self.committedChanges.add(int(details["change"]))
2216         committer = ""
2217         if author not in self.users:
2218             self.getUserMapFromPerforceServer()
2219         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2220
2221         self.gitStream.write("committer %s\n" % committer)
2222
2223         self.gitStream.write("data <<EOT\n")
2224         self.gitStream.write(details["desc"])
2225         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2226                              (','.join(self.branchPrefixes), details["change"]))
2227         if len(details['options']) > 0:
2228             self.gitStream.write(": options = %s" % details['options'])
2229         self.gitStream.write("]\nEOT\n\n")
2230
2231         if len(parent) > 0:
2232             if self.verbose:
2233                 print "parent %s" % parent
2234             self.gitStream.write("from %s\n" % parent)
2235
2236         self.streamP4Files(new_files)
2237         self.gitStream.write("\n")
2238
2239         change = int(details["change"])
2240
2241         if self.labels.has_key(change):
2242             label = self.labels[change]
2243             labelDetails = label[0]
2244             labelRevisions = label[1]
2245             if self.verbose:
2246                 print "Change %s is labelled %s" % (change, labelDetails)
2247
2248             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2249                                                 for p in self.branchPrefixes])
2250
2251             if len(files) == len(labelRevisions):
2252
2253                 cleanedFiles = {}
2254                 for info in files:
2255                     if info["action"] in self.delete_actions:
2256                         continue
2257                     cleanedFiles[info["depotFile"]] = info["rev"]
2258
2259                 if cleanedFiles == labelRevisions:
2260                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2261
2262                 else:
2263                     if not self.silent:
2264                         print ("Tag %s does not match with change %s: files do not match."
2265                                % (labelDetails["label"], change))
2266
2267             else:
2268                 if not self.silent:
2269                     print ("Tag %s does not match with change %s: file count is different."
2270                            % (labelDetails["label"], change))
2271
2272     # Build a dictionary of changelists and labels, for "detect-labels" option.
2273     def getLabels(self):
2274         self.labels = {}
2275
2276         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2277         if len(l) > 0 and not self.silent:
2278             print "Finding files belonging to labels in %s" % `self.depotPaths`
2279
2280         for output in l:
2281             label = output["label"]
2282             revisions = {}
2283             newestChange = 0
2284             if self.verbose:
2285                 print "Querying files for label %s" % label
2286             for file in p4CmdList(["files"] +
2287                                       ["%s...@%s" % (p, label)
2288                                           for p in self.depotPaths]):
2289                 revisions[file["depotFile"]] = file["rev"]
2290                 change = int(file["change"])
2291                 if change > newestChange:
2292                     newestChange = change
2293
2294             self.labels[newestChange] = [output, revisions]
2295
2296         if self.verbose:
2297             print "Label changes: %s" % self.labels.keys()
2298
2299     # Import p4 labels as git tags. A direct mapping does not
2300     # exist, so assume that if all the files are at the same revision
2301     # then we can use that, or it's something more complicated we should
2302     # just ignore.
2303     def importP4Labels(self, stream, p4Labels):
2304         if verbose:
2305             print "import p4 labels: " + ' '.join(p4Labels)
2306
2307         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2308         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2309         if len(validLabelRegexp) == 0:
2310             validLabelRegexp = defaultLabelRegexp
2311         m = re.compile(validLabelRegexp)
2312
2313         for name in p4Labels:
2314             commitFound = False
2315
2316             if not m.match(name):
2317                 if verbose:
2318                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2319                 continue
2320
2321             if name in ignoredP4Labels:
2322                 continue
2323
2324             labelDetails = p4CmdList(['label', "-o", name])[0]
2325
2326             # get the most recent changelist for each file in this label
2327             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2328                                 for p in self.depotPaths])
2329
2330             if change.has_key('change'):
2331                 # find the corresponding git commit; take the oldest commit
2332                 changelist = int(change['change'])
2333                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2334                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2335                 if len(gitCommit) == 0:
2336                     print "could not find git commit for changelist %d" % changelist
2337                 else:
2338                     gitCommit = gitCommit.strip()
2339                     commitFound = True
2340                     # Convert from p4 time format
2341                     try:
2342                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2343                     except ValueError:
2344                         print "Could not convert label time %s" % labelDetail['Update']
2345                         tmwhen = 1
2346
2347                     when = int(time.mktime(tmwhen))
2348                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2349                     if verbose:
2350                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2351             else:
2352                 if verbose:
2353                     print "Label %s has no changelists - possibly deleted?" % name
2354
2355             if not commitFound:
2356                 # We can't import this label; don't try again as it will get very
2357                 # expensive repeatedly fetching all the files for labels that will
2358                 # never be imported. If the label is moved in the future, the
2359                 # ignore will need to be removed manually.
2360                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2361
2362     def guessProjectName(self):
2363         for p in self.depotPaths:
2364             if p.endswith("/"):
2365                 p = p[:-1]
2366             p = p[p.strip().rfind("/") + 1:]
2367             if not p.endswith("/"):
2368                p += "/"
2369             return p
2370
2371     def getBranchMapping(self):
2372         lostAndFoundBranches = set()
2373
2374         user = gitConfig("git-p4.branchUser")
2375         if len(user) > 0:
2376             command = "branches -u %s" % user
2377         else:
2378             command = "branches"
2379
2380         for info in p4CmdList(command):
2381             details = p4Cmd(["branch", "-o", info["branch"]])
2382             viewIdx = 0
2383             while details.has_key("View%s" % viewIdx):
2384                 paths = details["View%s" % viewIdx].split(" ")
2385                 viewIdx = viewIdx + 1
2386                 # require standard //depot/foo/... //depot/bar/... mapping
2387                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2388                     continue
2389                 source = paths[0]
2390                 destination = paths[1]
2391                 ## HACK
2392                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2393                     source = source[len(self.depotPaths[0]):-4]
2394                     destination = destination[len(self.depotPaths[0]):-4]
2395
2396                     if destination in self.knownBranches:
2397                         if not self.silent:
2398                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2399                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2400                         continue
2401
2402                     self.knownBranches[destination] = source
2403
2404                     lostAndFoundBranches.discard(destination)
2405
2406                     if source not in self.knownBranches:
2407                         lostAndFoundBranches.add(source)
2408
2409         # Perforce does not strictly require branches to be defined, so we also
2410         # check git config for a branch list.
2411         #
2412         # Example of branch definition in git config file:
2413         # [git-p4]
2414         #   branchList=main:branchA
2415         #   branchList=main:branchB
2416         #   branchList=branchA:branchC
2417         configBranches = gitConfigList("git-p4.branchList")
2418         for branch in configBranches:
2419             if branch:
2420                 (source, destination) = branch.split(":")
2421                 self.knownBranches[destination] = source
2422
2423                 lostAndFoundBranches.discard(destination)
2424
2425                 if source not in self.knownBranches:
2426                     lostAndFoundBranches.add(source)
2427
2428
2429         for branch in lostAndFoundBranches:
2430             self.knownBranches[branch] = branch
2431
2432     def getBranchMappingFromGitBranches(self):
2433         branches = p4BranchesInGit(self.importIntoRemotes)
2434         for branch in branches.keys():
2435             if branch == "master":
2436                 branch = "main"
2437             else:
2438                 branch = branch[len(self.projectName):]
2439             self.knownBranches[branch] = branch
2440
2441     def listExistingP4GitBranches(self):
2442         # branches holds mapping from name to commit
2443         branches = p4BranchesInGit(self.importIntoRemotes)
2444         self.p4BranchesInGit = branches.keys()
2445         for branch in branches.keys():
2446             self.initialParents[self.refPrefix + branch] = branches[branch]
2447
2448     def updateOptionDict(self, d):
2449         option_keys = {}
2450         if self.keepRepoPath:
2451             option_keys['keepRepoPath'] = 1
2452
2453         d["options"] = ' '.join(sorted(option_keys.keys()))
2454
2455     def readOptions(self, d):
2456         self.keepRepoPath = (d.has_key('options')
2457                              and ('keepRepoPath' in d['options']))
2458
2459     def gitRefForBranch(self, branch):
2460         if branch == "main":
2461             return self.refPrefix + "master"
2462
2463         if len(branch) <= 0:
2464             return branch
2465
2466         return self.refPrefix + self.projectName + branch
2467
2468     def gitCommitByP4Change(self, ref, change):
2469         if self.verbose:
2470             print "looking in ref " + ref + " for change %s using bisect..." % change
2471
2472         earliestCommit = ""
2473         latestCommit = parseRevision(ref)
2474
2475         while True:
2476             if self.verbose:
2477                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2478             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2479             if len(next) == 0:
2480                 if self.verbose:
2481                     print "argh"
2482                 return ""
2483             log = extractLogMessageFromGitCommit(next)
2484             settings = extractSettingsGitLog(log)
2485             currentChange = int(settings['change'])
2486             if self.verbose:
2487                 print "current change %s" % currentChange
2488
2489             if currentChange == change:
2490                 if self.verbose:
2491                     print "found %s" % next
2492                 return next
2493
2494             if currentChange < change:
2495                 earliestCommit = "^%s" % next
2496             else:
2497                 latestCommit = "%s" % next
2498
2499         return ""
2500
2501     def importNewBranch(self, branch, maxChange):
2502         # make fast-import flush all changes to disk and update the refs using the checkpoint
2503         # command so that we can try to find the branch parent in the git history
2504         self.gitStream.write("checkpoint\n\n");
2505         self.gitStream.flush();
2506         branchPrefix = self.depotPaths[0] + branch + "/"
2507         range = "@1,%s" % maxChange
2508         #print "prefix" + branchPrefix
2509         changes = p4ChangesForPaths([branchPrefix], range)
2510         if len(changes) <= 0:
2511             return False
2512         firstChange = changes[0]
2513         #print "first change in branch: %s" % firstChange
2514         sourceBranch = self.knownBranches[branch]
2515         sourceDepotPath = self.depotPaths[0] + sourceBranch
2516         sourceRef = self.gitRefForBranch(sourceBranch)
2517         #print "source " + sourceBranch
2518
2519         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2520         #print "branch parent: %s" % branchParentChange
2521         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2522         if len(gitParent) > 0:
2523             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2524             #print "parent git commit: %s" % gitParent
2525
2526         self.importChanges(changes)
2527         return True
2528
2529     def searchParent(self, parent, branch, target):
2530         parentFound = False
2531         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2532             blob = blob.strip()
2533             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2534                 parentFound = True
2535                 if self.verbose:
2536                     print "Found parent of %s in commit %s" % (branch, blob)
2537                 break
2538         if parentFound:
2539             return blob
2540         else:
2541             return None
2542
2543     def importChanges(self, changes):
2544         cnt = 1
2545         for change in changes:
2546             description = p4Cmd(["describe", str(change)])
2547             self.updateOptionDict(description)
2548
2549             if not self.silent:
2550                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2551                 sys.stdout.flush()
2552             cnt = cnt + 1
2553
2554             try:
2555                 if self.detectBranches:
2556                     branches = self.splitFilesIntoBranches(description)
2557                     for branch in branches.keys():
2558                         ## HACK  --hwn
2559                         branchPrefix = self.depotPaths[0] + branch + "/"
2560                         self.branchPrefixes = [ branchPrefix ]
2561
2562                         parent = ""
2563
2564                         filesForCommit = branches[branch]
2565
2566                         if self.verbose:
2567                             print "branch is %s" % branch
2568
2569                         self.updatedBranches.add(branch)
2570
2571                         if branch not in self.createdBranches:
2572                             self.createdBranches.add(branch)
2573                             parent = self.knownBranches[branch]
2574                             if parent == branch:
2575                                 parent = ""
2576                             else:
2577                                 fullBranch = self.projectName + branch
2578                                 if fullBranch not in self.p4BranchesInGit:
2579                                     if not self.silent:
2580                                         print("\n    Importing new branch %s" % fullBranch);
2581                                     if self.importNewBranch(branch, change - 1):
2582                                         parent = ""
2583                                         self.p4BranchesInGit.append(fullBranch)
2584                                     if not self.silent:
2585                                         print("\n    Resuming with change %s" % change);
2586
2587                                 if self.verbose:
2588                                     print "parent determined through known branches: %s" % parent
2589
2590                         branch = self.gitRefForBranch(branch)
2591                         parent = self.gitRefForBranch(parent)
2592
2593                         if self.verbose:
2594                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2595
2596                         if len(parent) == 0 and branch in self.initialParents:
2597                             parent = self.initialParents[branch]
2598                             del self.initialParents[branch]
2599
2600                         blob = None
2601                         if len(parent) > 0:
2602                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2603                             if self.verbose:
2604                                 print "Creating temporary branch: " + tempBranch
2605                             self.commit(description, filesForCommit, tempBranch)
2606                             self.tempBranches.append(tempBranch)
2607                             self.checkpoint()
2608                             blob = self.searchParent(parent, branch, tempBranch)
2609                         if blob:
2610                             self.commit(description, filesForCommit, branch, blob)
2611                         else:
2612                             if self.verbose:
2613                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2614                             self.commit(description, filesForCommit, branch, parent)
2615                 else:
2616                     files = self.extractFilesFromCommit(description)
2617                     self.commit(description, files, self.branch,
2618                                 self.initialParent)
2619                     self.initialParent = ""
2620             except IOError:
2621                 print self.gitError.read()
2622                 sys.exit(1)
2623
2624     def importHeadRevision(self, revision):
2625         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2626
2627         details = {}
2628         details["user"] = "git perforce import user"
2629         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2630                            % (' '.join(self.depotPaths), revision))
2631         details["change"] = revision
2632         newestRevision = 0
2633
2634         fileCnt = 0
2635         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2636
2637         for info in p4CmdList(["files"] + fileArgs):
2638
2639             if 'code' in info and info['code'] == 'error':
2640                 sys.stderr.write("p4 returned an error: %s\n"
2641                                  % info['data'])
2642                 if info['data'].find("must refer to client") >= 0:
2643                     sys.stderr.write("This particular p4 error is misleading.\n")
2644                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2645                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2646                 sys.exit(1)
2647             if 'p4ExitCode' in info:
2648                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2649                 sys.exit(1)
2650
2651
2652             change = int(info["change"])
2653             if change > newestRevision:
2654                 newestRevision = change
2655
2656             if info["action"] in self.delete_actions:
2657                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2658                 #fileCnt = fileCnt + 1
2659                 continue
2660
2661             for prop in ["depotFile", "rev", "action", "type" ]:
2662                 details["%s%s" % (prop, fileCnt)] = info[prop]
2663
2664             fileCnt = fileCnt + 1
2665
2666         details["change"] = newestRevision
2667
2668         # Use time from top-most change so that all git p4 clones of
2669         # the same p4 repo have the same commit SHA1s.
2670         res = p4CmdList("describe -s %d" % newestRevision)
2671         newestTime = None
2672         for r in res:
2673             if r.has_key('time'):
2674                 newestTime = int(r['time'])
2675         if newestTime is None:
2676             die("\"describe -s\" on newest change %d did not give a time")
2677         details["time"] = newestTime
2678
2679         self.updateOptionDict(details)
2680         try:
2681             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2682         except IOError:
2683             print "IO error with git fast-import. Is your git version recent enough?"
2684             print self.gitError.read()
2685
2686
2687     def run(self, args):
2688         self.depotPaths = []
2689         self.changeRange = ""
2690         self.initialParent = ""
2691         self.previousDepotPaths = []
2692
2693         # map from branch depot path to parent branch
2694         self.knownBranches = {}
2695         self.initialParents = {}
2696         self.hasOrigin = originP4BranchesExist()
2697         if not self.syncWithOrigin:
2698             self.hasOrigin = False
2699
2700         if self.importIntoRemotes:
2701             self.refPrefix = "refs/remotes/p4/"
2702         else:
2703             self.refPrefix = "refs/heads/p4/"
2704
2705         if self.syncWithOrigin and self.hasOrigin:
2706             if not self.silent:
2707                 print "Syncing with origin first by calling git fetch origin"
2708             system("git fetch origin")
2709
2710         if len(self.branch) == 0:
2711             self.branch = self.refPrefix + "master"
2712             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2713                 system("git update-ref %s refs/heads/p4" % self.branch)
2714                 system("git branch -D p4");
2715             # create it /after/ importing, when master exists
2716             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2717                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2718
2719         # accept either the command-line option, or the configuration variable
2720         if self.useClientSpec:
2721             # will use this after clone to set the variable
2722             self.useClientSpec_from_options = True
2723         else:
2724             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2725                 self.useClientSpec = True
2726         if self.useClientSpec:
2727             self.clientSpecDirs = getClientSpec()
2728
2729         # TODO: should always look at previous commits,
2730         # merge with previous imports, if possible.
2731         if args == []:
2732             if self.hasOrigin:
2733                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2734             self.listExistingP4GitBranches()
2735
2736             if len(self.p4BranchesInGit) > 1:
2737                 if not self.silent:
2738                     print "Importing from/into multiple branches"
2739                 self.detectBranches = True
2740
2741             if self.verbose:
2742                 print "branches: %s" % self.p4BranchesInGit
2743
2744             p4Change = 0
2745             for branch in self.p4BranchesInGit:
2746                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2747
2748                 settings = extractSettingsGitLog(logMsg)
2749
2750                 self.readOptions(settings)
2751                 if (settings.has_key('depot-paths')
2752                     and settings.has_key ('change')):
2753                     change = int(settings['change']) + 1
2754                     p4Change = max(p4Change, change)
2755
2756                     depotPaths = sorted(settings['depot-paths'])
2757                     if self.previousDepotPaths == []:
2758                         self.previousDepotPaths = depotPaths
2759                     else:
2760                         paths = []
2761                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2762                             prev_list = prev.split("/")
2763                             cur_list = cur.split("/")
2764                             for i in range(0, min(len(cur_list), len(prev_list))):
2765                                 if cur_list[i] <> prev_list[i]:
2766                                     i = i - 1
2767                                     break
2768
2769                             paths.append ("/".join(cur_list[:i + 1]))
2770
2771                         self.previousDepotPaths = paths
2772
2773             if p4Change > 0:
2774                 self.depotPaths = sorted(self.previousDepotPaths)
2775                 self.changeRange = "@%s,#head" % p4Change
2776                 if not self.detectBranches:
2777                     self.initialParent = parseRevision(self.branch)
2778                 if not self.silent and not self.detectBranches:
2779                     print "Performing incremental import into %s git branch" % self.branch
2780
2781         if not self.branch.startswith("refs/"):
2782             self.branch = "refs/heads/" + self.branch
2783
2784         if len(args) == 0 and self.depotPaths:
2785             if not self.silent:
2786                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2787         else:
2788             if self.depotPaths and self.depotPaths != args:
2789                 print ("previous import used depot path %s and now %s was specified. "
2790                        "This doesn't work!" % (' '.join (self.depotPaths),
2791                                                ' '.join (args)))
2792                 sys.exit(1)
2793
2794             self.depotPaths = sorted(args)
2795
2796         revision = ""
2797         self.users = {}
2798
2799         # Make sure no revision specifiers are used when --changesfile
2800         # is specified.
2801         bad_changesfile = False
2802         if len(self.changesFile) > 0:
2803             for p in self.depotPaths:
2804                 if p.find("@") >= 0 or p.find("#") >= 0:
2805                     bad_changesfile = True
2806                     break
2807         if bad_changesfile:
2808             die("Option --changesfile is incompatible with revision specifiers")
2809
2810         newPaths = []
2811         for p in self.depotPaths:
2812             if p.find("@") != -1:
2813                 atIdx = p.index("@")
2814                 self.changeRange = p[atIdx:]
2815                 if self.changeRange == "@all":
2816                     self.changeRange = ""
2817                 elif ',' not in self.changeRange:
2818                     revision = self.changeRange
2819                     self.changeRange = ""
2820                 p = p[:atIdx]
2821             elif p.find("#") != -1:
2822                 hashIdx = p.index("#")
2823                 revision = p[hashIdx:]
2824                 p = p[:hashIdx]
2825             elif self.previousDepotPaths == []:
2826                 # pay attention to changesfile, if given, else import
2827                 # the entire p4 tree at the head revision
2828                 if len(self.changesFile) == 0:
2829                     revision = "#head"
2830
2831             p = re.sub ("\.\.\.$", "", p)
2832             if not p.endswith("/"):
2833                 p += "/"
2834
2835             newPaths.append(p)
2836
2837         self.depotPaths = newPaths
2838
2839         # --detect-branches may change this for each branch
2840         self.branchPrefixes = self.depotPaths
2841
2842         self.loadUserMapFromCache()
2843         self.labels = {}
2844         if self.detectLabels:
2845             self.getLabels();
2846
2847         if self.detectBranches:
2848             ## FIXME - what's a P4 projectName ?
2849             self.projectName = self.guessProjectName()
2850
2851             if self.hasOrigin:
2852                 self.getBranchMappingFromGitBranches()
2853             else:
2854                 self.getBranchMapping()
2855             if self.verbose:
2856                 print "p4-git branches: %s" % self.p4BranchesInGit
2857                 print "initial parents: %s" % self.initialParents
2858             for b in self.p4BranchesInGit:
2859                 if b != "master":
2860
2861                     ## FIXME
2862                     b = b[len(self.projectName):]
2863                 self.createdBranches.add(b)
2864
2865         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2866
2867         importProcess = subprocess.Popen(["git", "fast-import"],
2868                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2869                                          stderr=subprocess.PIPE);
2870         self.gitOutput = importProcess.stdout
2871         self.gitStream = importProcess.stdin
2872         self.gitError = importProcess.stderr
2873
2874         if revision:
2875             self.importHeadRevision(revision)
2876         else:
2877             changes = []
2878
2879             if len(self.changesFile) > 0:
2880                 output = open(self.changesFile).readlines()
2881                 changeSet = set()
2882                 for line in output:
2883                     changeSet.add(int(line))
2884
2885                 for change in changeSet:
2886                     changes.append(change)
2887
2888                 changes.sort()
2889             else:
2890                 # catch "git p4 sync" with no new branches, in a repo that
2891                 # does not have any existing p4 branches
2892                 if len(args) == 0 and not self.p4BranchesInGit:
2893                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2894                 if self.verbose:
2895                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2896                                                               self.changeRange)
2897                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2898
2899                 if len(self.maxChanges) > 0:
2900                     changes = changes[:min(int(self.maxChanges), len(changes))]
2901
2902             if len(changes) == 0:
2903                 if not self.silent:
2904                     print "No changes to import!"
2905             else:
2906                 if not self.silent and not self.detectBranches:
2907                     print "Import destination: %s" % self.branch
2908
2909                 self.updatedBranches = set()
2910
2911                 self.importChanges(changes)
2912
2913                 if not self.silent:
2914                     print ""
2915                     if len(self.updatedBranches) > 0:
2916                         sys.stdout.write("Updated branches: ")
2917                         for b in self.updatedBranches:
2918                             sys.stdout.write("%s " % b)
2919                         sys.stdout.write("\n")
2920
2921         if gitConfig("git-p4.importLabels", "--bool") == "true":
2922             self.importLabels = True
2923
2924         if self.importLabels:
2925             p4Labels = getP4Labels(self.depotPaths)
2926             gitTags = getGitTags()
2927
2928             missingP4Labels = p4Labels - gitTags
2929             self.importP4Labels(self.gitStream, missingP4Labels)
2930
2931         self.gitStream.close()
2932         if importProcess.wait() != 0:
2933             die("fast-import failed: %s" % self.gitError.read())
2934         self.gitOutput.close()
2935         self.gitError.close()
2936
2937         # Cleanup temporary branches created during import
2938         if self.tempBranches != []:
2939             for branch in self.tempBranches:
2940                 read_pipe("git update-ref -d %s" % branch)
2941             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2942
2943         return True
2944
2945 class P4Rebase(Command):
2946     def __init__(self):
2947         Command.__init__(self)
2948         self.options = [
2949                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2950         ]
2951         self.importLabels = False
2952         self.description = ("Fetches the latest revision from perforce and "
2953                             + "rebases the current work (branch) against it")
2954
2955     def run(self, args):
2956         sync = P4Sync()
2957         sync.importLabels = self.importLabels
2958         sync.run([])
2959
2960         return self.rebase()
2961
2962     def rebase(self):
2963         if os.system("git update-index --refresh") != 0:
2964             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2965         if len(read_pipe("git diff-index HEAD --")) > 0:
2966             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2967
2968         [upstream, settings] = findUpstreamBranchPoint()
2969         if len(upstream) == 0:
2970             die("Cannot find upstream branchpoint for rebase")
2971
2972         # the branchpoint may be p4/foo~3, so strip off the parent
2973         upstream = re.sub("~[0-9]+$", "", upstream)
2974
2975         print "Rebasing the current branch onto %s" % upstream
2976         oldHead = read_pipe("git rev-parse HEAD").strip()
2977         system("git rebase %s" % upstream)
2978         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2979         return True
2980
2981 class P4Clone(P4Sync):
2982     def __init__(self):
2983         P4Sync.__init__(self)
2984         self.description = "Creates a new git repository and imports from Perforce into it"
2985         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2986         self.options += [
2987             optparse.make_option("--destination", dest="cloneDestination",
2988                                  action='store', default=None,
2989                                  help="where to leave result of the clone"),
2990             optparse.make_option("-/", dest="cloneExclude",
2991                                  action="append", type="string",
2992                                  help="exclude depot path"),
2993             optparse.make_option("--bare", dest="cloneBare",
2994                                  action="store_true", default=False),
2995         ]
2996         self.cloneDestination = None
2997         self.needsGit = False
2998         self.cloneBare = False
2999
3000     # This is required for the "append" cloneExclude action
3001     def ensure_value(self, attr, value):
3002         if not hasattr(self, attr) or getattr(self, attr) is None:
3003             setattr(self, attr, value)
3004         return getattr(self, attr)
3005
3006     def defaultDestination(self, args):
3007         ## TODO: use common prefix of args?
3008         depotPath = args[0]
3009         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3010         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3011         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3012         depotDir = re.sub(r"/$", "", depotDir)
3013         return os.path.split(depotDir)[1]
3014
3015     def run(self, args):
3016         if len(args) < 1:
3017             return False
3018
3019         if self.keepRepoPath and not self.cloneDestination:
3020             sys.stderr.write("Must specify destination for --keep-path\n")
3021             sys.exit(1)
3022
3023         depotPaths = args
3024
3025         if not self.cloneDestination and len(depotPaths) > 1:
3026             self.cloneDestination = depotPaths[-1]
3027             depotPaths = depotPaths[:-1]
3028
3029         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3030         for p in depotPaths:
3031             if not p.startswith("//"):
3032                 return False
3033
3034         if not self.cloneDestination:
3035             self.cloneDestination = self.defaultDestination(args)
3036
3037         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3038
3039         if not os.path.exists(self.cloneDestination):
3040             os.makedirs(self.cloneDestination)
3041         chdir(self.cloneDestination)
3042
3043         init_cmd = [ "git", "init" ]
3044         if self.cloneBare:
3045             init_cmd.append("--bare")
3046         subprocess.check_call(init_cmd)
3047
3048         if not P4Sync.run(self, depotPaths):
3049             return False
3050         if self.branch != "master":
3051             if self.importIntoRemotes:
3052                 masterbranch = "refs/remotes/p4/master"
3053             else:
3054                 masterbranch = "refs/heads/p4/master"
3055             if gitBranchExists(masterbranch):
3056                 system("git branch master %s" % masterbranch)
3057                 if not self.cloneBare:
3058                     system("git checkout -f")
3059             else:
3060                 print "Could not detect main branch. No checkout/master branch created."
3061
3062         # auto-set this variable if invoked with --use-client-spec
3063         if self.useClientSpec_from_options:
3064             system("git config --bool git-p4.useclientspec true")
3065
3066         return True
3067
3068 class P4Branches(Command):
3069     def __init__(self):
3070         Command.__init__(self)
3071         self.options = [ ]
3072         self.description = ("Shows the git branches that hold imports and their "
3073                             + "corresponding perforce depot paths")
3074         self.verbose = False
3075
3076     def run(self, args):
3077         if originP4BranchesExist():
3078             createOrUpdateBranchesFromOrigin()
3079
3080         cmdline = "git rev-parse --symbolic "
3081         cmdline += " --remotes"
3082
3083         for line in read_pipe_lines(cmdline):
3084             line = line.strip()
3085
3086             if not line.startswith('p4/') or line == "p4/HEAD":
3087                 continue
3088             branch = line
3089
3090             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3091             settings = extractSettingsGitLog(log)
3092
3093             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3094         return True
3095
3096 class HelpFormatter(optparse.IndentedHelpFormatter):
3097     def __init__(self):
3098         optparse.IndentedHelpFormatter.__init__(self)
3099
3100     def format_description(self, description):
3101         if description:
3102             return description + "\n"
3103         else:
3104             return ""
3105
3106 def printUsage(commands):
3107     print "usage: %s <command> [options]" % sys.argv[0]
3108     print ""
3109     print "valid commands: %s" % ", ".join(commands)
3110     print ""
3111     print "Try %s <command> --help for command specific help." % sys.argv[0]
3112     print ""
3113
3114 commands = {
3115     "debug" : P4Debug,
3116     "submit" : P4Submit,
3117     "commit" : P4Submit,
3118     "sync" : P4Sync,
3119     "rebase" : P4Rebase,
3120     "clone" : P4Clone,
3121     "rollback" : P4RollBack,
3122     "branches" : P4Branches
3123 }
3124
3125
3126 def main():
3127     if len(sys.argv[1:]) == 0:
3128         printUsage(commands.keys())
3129         sys.exit(2)
3130
3131     cmd = ""
3132     cmdName = sys.argv[1]
3133     try:
3134         klass = commands[cmdName]
3135         cmd = klass()
3136     except KeyError:
3137         print "unknown command %s" % cmdName
3138         print ""
3139         printUsage(commands.keys())
3140         sys.exit(2)
3141
3142     options = cmd.options
3143     cmd.gitdir = os.environ.get("GIT_DIR", None)
3144
3145     args = sys.argv[2:]
3146
3147     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3148     if cmd.needsGit:
3149         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3150
3151     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3152                                    options,
3153                                    description = cmd.description,
3154                                    formatter = HelpFormatter())
3155
3156     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3157     global verbose
3158     verbose = cmd.verbose
3159     if cmd.needsGit:
3160         if cmd.gitdir == None:
3161             cmd.gitdir = os.path.abspath(".git")
3162             if not isValidGitDir(cmd.gitdir):
3163                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3164                 if os.path.exists(cmd.gitdir):
3165                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3166                     if len(cdup) > 0:
3167                         chdir(cdup);
3168
3169         if not isValidGitDir(cmd.gitdir):
3170             if isValidGitDir(cmd.gitdir + "/.git"):
3171                 cmd.gitdir += "/.git"
3172             else:
3173                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3174
3175         os.environ["GIT_DIR"] = cmd.gitdir
3176
3177     if not cmd.run(args):
3178         parser.print_help()
3179         sys.exit(2)
3180
3181
3182 if __name__ == '__main__':
3183     main()