Merge branch 'rr/precompose-utf8-cleanup' into maint
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 import re, shutil
14
15 verbose = False
16
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
19
20 def p4_build_cmd(cmd):
21     """Build a suitable p4 command line.
22
23     This consolidates building and returning a p4 command line into one
24     location. It means that hooking into the environment, or other configuration
25     can be done more easily.
26     """
27     real_cmd = ["p4"]
28
29     user = gitConfig("git-p4.user")
30     if len(user) > 0:
31         real_cmd += ["-u",user]
32
33     password = gitConfig("git-p4.password")
34     if len(password) > 0:
35         real_cmd += ["-P", password]
36
37     port = gitConfig("git-p4.port")
38     if len(port) > 0:
39         real_cmd += ["-p", port]
40
41     host = gitConfig("git-p4.host")
42     if len(host) > 0:
43         real_cmd += ["-H", host]
44
45     client = gitConfig("git-p4.client")
46     if len(client) > 0:
47         real_cmd += ["-c", client]
48
49
50     if isinstance(cmd,basestring):
51         real_cmd = ' '.join(real_cmd) + ' ' + cmd
52     else:
53         real_cmd += cmd
54     return real_cmd
55
56 def chdir(dir):
57     # P4 uses the PWD environment variable rather than getcwd(). Since we're
58     # not using the shell, we have to set it ourselves.  This path could
59     # be relative, so go there first, then figure out where we ended up.
60     os.chdir(dir)
61     os.environ['PWD'] = os.getcwd()
62
63 def die(msg):
64     if verbose:
65         raise Exception(msg)
66     else:
67         sys.stderr.write(msg + "\n")
68         sys.exit(1)
69
70 def write_pipe(c, stdin):
71     if verbose:
72         sys.stderr.write('Writing pipe: %s\n' % str(c))
73
74     expand = isinstance(c,basestring)
75     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
76     pipe = p.stdin
77     val = pipe.write(stdin)
78     pipe.close()
79     if p.wait():
80         die('Command failed: %s' % str(c))
81
82     return val
83
84 def p4_write_pipe(c, stdin):
85     real_cmd = p4_build_cmd(c)
86     return write_pipe(real_cmd, stdin)
87
88 def read_pipe(c, ignore_error=False):
89     if verbose:
90         sys.stderr.write('Reading pipe: %s\n' % str(c))
91
92     expand = isinstance(c,basestring)
93     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
94     pipe = p.stdout
95     val = pipe.read()
96     if p.wait() and not ignore_error:
97         die('Command failed: %s' % str(c))
98
99     return val
100
101 def p4_read_pipe(c, ignore_error=False):
102     real_cmd = p4_build_cmd(c)
103     return read_pipe(real_cmd, ignore_error)
104
105 def read_pipe_lines(c):
106     if verbose:
107         sys.stderr.write('Reading pipe: %s\n' % str(c))
108
109     expand = isinstance(c, basestring)
110     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
111     pipe = p.stdout
112     val = pipe.readlines()
113     if pipe.close() or p.wait():
114         die('Command failed: %s' % str(c))
115
116     return val
117
118 def p4_read_pipe_lines(c):
119     """Specifically invoke p4 on the command supplied. """
120     real_cmd = p4_build_cmd(c)
121     return read_pipe_lines(real_cmd)
122
123 def p4_has_command(cmd):
124     """Ask p4 for help on this command.  If it returns an error, the
125        command does not exist in this version of p4."""
126     real_cmd = p4_build_cmd(["help", cmd])
127     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
128                                    stderr=subprocess.PIPE)
129     p.communicate()
130     return p.returncode == 0
131
132 def system(cmd):
133     expand = isinstance(cmd,basestring)
134     if verbose:
135         sys.stderr.write("executing %s\n" % str(cmd))
136     subprocess.check_call(cmd, shell=expand)
137
138 def p4_system(cmd):
139     """Specifically invoke p4 as the system command. """
140     real_cmd = p4_build_cmd(cmd)
141     expand = isinstance(real_cmd, basestring)
142     subprocess.check_call(real_cmd, shell=expand)
143
144 def p4_integrate(src, dest):
145     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
146
147 def p4_sync(f, *options):
148     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
149
150 def p4_add(f):
151     # forcibly add file names with wildcards
152     if wildcard_present(f):
153         p4_system(["add", "-f", f])
154     else:
155         p4_system(["add", f])
156
157 def p4_delete(f):
158     p4_system(["delete", wildcard_encode(f)])
159
160 def p4_edit(f):
161     p4_system(["edit", wildcard_encode(f)])
162
163 def p4_revert(f):
164     p4_system(["revert", wildcard_encode(f)])
165
166 def p4_reopen(type, f):
167     p4_system(["reopen", "-t", type, wildcard_encode(f)])
168
169 def p4_move(src, dest):
170     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
171
172 #
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers.  See "p4 help filetypes"
175 # for a list and explanation.
176 #
177 def split_p4_type(p4type):
178
179     p4_filetypes_historical = {
180         "ctempobj": "binary+Sw",
181         "ctext": "text+C",
182         "cxtext": "text+Cx",
183         "ktext": "text+k",
184         "kxtext": "text+kx",
185         "ltext": "text+F",
186         "tempobj": "binary+FSw",
187         "ubinary": "binary+F",
188         "uresource": "resource+F",
189         "uxbinary": "binary+Fx",
190         "xbinary": "binary+x",
191         "xltext": "text+Fx",
192         "xtempobj": "binary+Swx",
193         "xtext": "text+x",
194         "xunicode": "unicode+x",
195         "xutf16": "utf16+x",
196     }
197     if p4type in p4_filetypes_historical:
198         p4type = p4_filetypes_historical[p4type]
199     mods = ""
200     s = p4type.split("+")
201     base = s[0]
202     mods = ""
203     if len(s) > 1:
204         mods = s[1]
205     return (base, mods)
206
207 #
208 # return the raw p4 type of a file (text, text+ko, etc)
209 #
210 def p4_type(file):
211     results = p4CmdList(["fstat", "-T", "headType", file])
212     return results[0]['headType']
213
214 #
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
217 #
218 def p4_keywords_regexp_for_type(base, type_mods):
219     if base in ("text", "unicode", "binary"):
220         kwords = None
221         if "ko" in type_mods:
222             kwords = 'Id|Header'
223         elif "k" in type_mods:
224             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
225         else:
226             return None
227         pattern = r"""
228             \$              # Starts with a dollar, followed by...
229             (%s)            # one of the keywords, followed by...
230             (:[^$]+)?       # possibly an old expansion, followed by...
231             \$              # another dollar
232             """ % kwords
233         return pattern
234     else:
235         return None
236
237 #
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
241 #
242 def p4_keywords_regexp_for_file(file):
243     if not os.path.exists(file):
244         return None
245     else:
246         (type_base, type_mods) = split_p4_type(p4_type(file))
247         return p4_keywords_regexp_for_type(type_base, type_mods)
248
249 def setP4ExecBit(file, mode):
250     # Reopens an already open file and changes the execute bit to match
251     # the execute bit setting in the passed in mode.
252
253     p4Type = "+x"
254
255     if not isModeExec(mode):
256         p4Type = getP4OpenedType(file)
257         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
258         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
259         if p4Type[-1] == "+":
260             p4Type = p4Type[0:-1]
261
262     p4_reopen(p4Type, file)
263
264 def getP4OpenedType(file):
265     # Returns the perforce file type for the given file.
266
267     result = p4_read_pipe(["opened", wildcard_encode(file)])
268     match = re.match(".*\((.+)\)\r?$", result)
269     if match:
270         return match.group(1)
271     else:
272         die("Could not determine file type for %s (result: '%s')" % (file, result))
273
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths):
276     labels = set()
277     if isinstance(depotPaths,basestring):
278         depotPaths = [depotPaths]
279
280     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
281         label = l['label']
282         labels.add(label)
283
284     return labels
285
286 # Return the set of all git tags
287 def getGitTags():
288     gitTags = set()
289     for line in read_pipe_lines(["git", "tag"]):
290         tag = line.strip()
291         gitTags.add(tag)
292     return gitTags
293
294 def diffTreePattern():
295     # This is a simple generator for the diff tree regex pattern. This could be
296     # a class variable if this and parseDiffTreeEntry were a part of a class.
297     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
298     while True:
299         yield pattern
300
301 def parseDiffTreeEntry(entry):
302     """Parses a single diff tree entry into its component elements.
303
304     See git-diff-tree(1) manpage for details about the format of the diff
305     output. This method returns a dictionary with the following elements:
306
307     src_mode - The mode of the source file
308     dst_mode - The mode of the destination file
309     src_sha1 - The sha1 for the source file
310     dst_sha1 - The sha1 fr the destination file
311     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312     status_score - The score for the status (applicable for 'C' and 'R'
313                    statuses). This is None if there is no score.
314     src - The path for the source file.
315     dst - The path for the destination file. This is only present for
316           copy or renames. If it is not present, this is None.
317
318     If the pattern is not matched, None is returned."""
319
320     match = diffTreePattern().next().match(entry)
321     if match:
322         return {
323             'src_mode': match.group(1),
324             'dst_mode': match.group(2),
325             'src_sha1': match.group(3),
326             'dst_sha1': match.group(4),
327             'status': match.group(5),
328             'status_score': match.group(6),
329             'src': match.group(7),
330             'dst': match.group(10)
331         }
332     return None
333
334 def isModeExec(mode):
335     # Returns True if the given git mode represents an executable file,
336     # otherwise False.
337     return mode[-3:] == "755"
338
339 def isModeExecChanged(src_mode, dst_mode):
340     return isModeExec(src_mode) != isModeExec(dst_mode)
341
342 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
343
344     if isinstance(cmd,basestring):
345         cmd = "-G " + cmd
346         expand = True
347     else:
348         cmd = ["-G"] + cmd
349         expand = False
350
351     cmd = p4_build_cmd(cmd)
352     if verbose:
353         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
354
355     # Use a temporary file to avoid deadlocks without
356     # subprocess.communicate(), which would put another copy
357     # of stdout into memory.
358     stdin_file = None
359     if stdin is not None:
360         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
361         if isinstance(stdin,basestring):
362             stdin_file.write(stdin)
363         else:
364             for i in stdin:
365                 stdin_file.write(i + '\n')
366         stdin_file.flush()
367         stdin_file.seek(0)
368
369     p4 = subprocess.Popen(cmd,
370                           shell=expand,
371                           stdin=stdin_file,
372                           stdout=subprocess.PIPE)
373
374     result = []
375     try:
376         while True:
377             entry = marshal.load(p4.stdout)
378             if cb is not None:
379                 cb(entry)
380             else:
381                 result.append(entry)
382     except EOFError:
383         pass
384     exitCode = p4.wait()
385     if exitCode != 0:
386         entry = {}
387         entry["p4ExitCode"] = exitCode
388         result.append(entry)
389
390     return result
391
392 def p4Cmd(cmd):
393     list = p4CmdList(cmd)
394     result = {}
395     for entry in list:
396         result.update(entry)
397     return result;
398
399 def p4Where(depotPath):
400     if not depotPath.endswith("/"):
401         depotPath += "/"
402     depotPath = depotPath + "..."
403     outputList = p4CmdList(["where", depotPath])
404     output = None
405     for entry in outputList:
406         if "depotFile" in entry:
407             if entry["depotFile"] == depotPath:
408                 output = entry
409                 break
410         elif "data" in entry:
411             data = entry.get("data")
412             space = data.find(" ")
413             if data[:space] == depotPath:
414                 output = entry
415                 break
416     if output == None:
417         return ""
418     if output["code"] == "error":
419         return ""
420     clientPath = ""
421     if "path" in output:
422         clientPath = output.get("path")
423     elif "data" in output:
424         data = output.get("data")
425         lastSpace = data.rfind(" ")
426         clientPath = data[lastSpace + 1:]
427
428     if clientPath.endswith("..."):
429         clientPath = clientPath[:-3]
430     return clientPath
431
432 def currentGitBranch():
433     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
434
435 def isValidGitDir(path):
436     if (os.path.exists(path + "/HEAD")
437         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
438         return True;
439     return False
440
441 def parseRevision(ref):
442     return read_pipe("git rev-parse %s" % ref).strip()
443
444 def branchExists(ref):
445     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
446                      ignore_error=True)
447     return len(rev) > 0
448
449 def extractLogMessageFromGitCommit(commit):
450     logMessage = ""
451
452     ## fixme: title is first line of commit, not 1st paragraph.
453     foundTitle = False
454     for log in read_pipe_lines("git cat-file commit %s" % commit):
455        if not foundTitle:
456            if len(log) == 1:
457                foundTitle = True
458            continue
459
460        logMessage += log
461     return logMessage
462
463 def extractSettingsGitLog(log):
464     values = {}
465     for line in log.split("\n"):
466         line = line.strip()
467         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
468         if not m:
469             continue
470
471         assignments = m.group(1).split (':')
472         for a in assignments:
473             vals = a.split ('=')
474             key = vals[0].strip()
475             val = ('='.join (vals[1:])).strip()
476             if val.endswith ('\"') and val.startswith('"'):
477                 val = val[1:-1]
478
479             values[key] = val
480
481     paths = values.get("depot-paths")
482     if not paths:
483         paths = values.get("depot-path")
484     if paths:
485         values['depot-paths'] = paths.split(',')
486     return values
487
488 def gitBranchExists(branch):
489     proc = subprocess.Popen(["git", "rev-parse", branch],
490                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
491     return proc.wait() == 0;
492
493 _gitConfig = {}
494 def gitConfig(key, args = None): # set args to "--bool", for instance
495     if not _gitConfig.has_key(key):
496         argsFilter = ""
497         if args != None:
498             argsFilter = "%s " % args
499         cmd = "git config %s%s" % (argsFilter, key)
500         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
501     return _gitConfig[key]
502
503 def gitConfigList(key):
504     if not _gitConfig.has_key(key):
505         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
506     return _gitConfig[key]
507
508 def p4BranchesInGit(branchesAreInRemotes = True):
509     branches = {}
510
511     cmdline = "git rev-parse --symbolic "
512     if branchesAreInRemotes:
513         cmdline += " --remotes"
514     else:
515         cmdline += " --branches"
516
517     for line in read_pipe_lines(cmdline):
518         line = line.strip()
519
520         ## only import to p4/
521         if not line.startswith('p4/') or line == "p4/HEAD":
522             continue
523         branch = line
524
525         # strip off p4
526         branch = re.sub ("^p4/", "", line)
527
528         branches[branch] = parseRevision(line)
529     return branches
530
531 def findUpstreamBranchPoint(head = "HEAD"):
532     branches = p4BranchesInGit()
533     # map from depot-path to branch name
534     branchByDepotPath = {}
535     for branch in branches.keys():
536         tip = branches[branch]
537         log = extractLogMessageFromGitCommit(tip)
538         settings = extractSettingsGitLog(log)
539         if settings.has_key("depot-paths"):
540             paths = ",".join(settings["depot-paths"])
541             branchByDepotPath[paths] = "remotes/p4/" + branch
542
543     settings = None
544     parent = 0
545     while parent < 65535:
546         commit = head + "~%s" % parent
547         log = extractLogMessageFromGitCommit(commit)
548         settings = extractSettingsGitLog(log)
549         if settings.has_key("depot-paths"):
550             paths = ",".join(settings["depot-paths"])
551             if branchByDepotPath.has_key(paths):
552                 return [branchByDepotPath[paths], settings]
553
554         parent = parent + 1
555
556     return ["", settings]
557
558 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
559     if not silent:
560         print ("Creating/updating branch(es) in %s based on origin branch(es)"
561                % localRefPrefix)
562
563     originPrefix = "origin/p4/"
564
565     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
566         line = line.strip()
567         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
568             continue
569
570         headName = line[len(originPrefix):]
571         remoteHead = localRefPrefix + headName
572         originHead = line
573
574         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
575         if (not original.has_key('depot-paths')
576             or not original.has_key('change')):
577             continue
578
579         update = False
580         if not gitBranchExists(remoteHead):
581             if verbose:
582                 print "creating %s" % remoteHead
583             update = True
584         else:
585             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
586             if settings.has_key('change') > 0:
587                 if settings['depot-paths'] == original['depot-paths']:
588                     originP4Change = int(original['change'])
589                     p4Change = int(settings['change'])
590                     if originP4Change > p4Change:
591                         print ("%s (%s) is newer than %s (%s). "
592                                "Updating p4 branch from origin."
593                                % (originHead, originP4Change,
594                                   remoteHead, p4Change))
595                         update = True
596                 else:
597                     print ("Ignoring: %s was imported from %s while "
598                            "%s was imported from %s"
599                            % (originHead, ','.join(original['depot-paths']),
600                               remoteHead, ','.join(settings['depot-paths'])))
601
602         if update:
603             system("git update-ref %s %s" % (remoteHead, originHead))
604
605 def originP4BranchesExist():
606         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
607
608 def p4ChangesForPaths(depotPaths, changeRange):
609     assert depotPaths
610     cmd = ['changes']
611     for p in depotPaths:
612         cmd += ["%s...%s" % (p, changeRange)]
613     output = p4_read_pipe_lines(cmd)
614
615     changes = {}
616     for line in output:
617         changeNum = int(line.split(" ")[1])
618         changes[changeNum] = True
619
620     changelist = changes.keys()
621     changelist.sort()
622     return changelist
623
624 def p4PathStartsWith(path, prefix):
625     # This method tries to remedy a potential mixed-case issue:
626     #
627     # If UserA adds  //depot/DirA/file1
628     # and UserB adds //depot/dira/file2
629     #
630     # we may or may not have a problem. If you have core.ignorecase=true,
631     # we treat DirA and dira as the same directory
632     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
633     if ignorecase:
634         return path.lower().startswith(prefix.lower())
635     return path.startswith(prefix)
636
637 def getClientSpec():
638     """Look at the p4 client spec, create a View() object that contains
639        all the mappings, and return it."""
640
641     specList = p4CmdList("client -o")
642     if len(specList) != 1:
643         die('Output from "client -o" is %d lines, expecting 1' %
644             len(specList))
645
646     # dictionary of all client parameters
647     entry = specList[0]
648
649     # just the keys that start with "View"
650     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
651
652     # hold this new View
653     view = View()
654
655     # append the lines, in order, to the view
656     for view_num in range(len(view_keys)):
657         k = "View%d" % view_num
658         if k not in view_keys:
659             die("Expected view key %s missing" % k)
660         view.append(entry[k])
661
662     return view
663
664 def getClientRoot():
665     """Grab the client directory."""
666
667     output = p4CmdList("client -o")
668     if len(output) != 1:
669         die('Output from "client -o" is %d lines, expecting 1' % len(output))
670
671     entry = output[0]
672     if "Root" not in entry:
673         die('Client has no "Root"')
674
675     return entry["Root"]
676
677 #
678 # P4 wildcards are not allowed in filenames.  P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
681 #
682 def wildcard_decode(path):
683     # Search for and fix just these four characters.  Do % last so
684     # that fixing it does not inadvertently create new %-escapes.
685     # Cannot have * in a filename in windows; untested as to
686     # what p4 would do in such a case.
687     if not platform.system() == "Windows":
688         path = path.replace("%2A", "*")
689     path = path.replace("%23", "#") \
690                .replace("%40", "@") \
691                .replace("%25", "%")
692     return path
693
694 def wildcard_encode(path):
695     # do % first to avoid double-encoding the %s introduced here
696     path = path.replace("%", "%25") \
697                .replace("*", "%2A") \
698                .replace("#", "%23") \
699                .replace("@", "%40")
700     return path
701
702 def wildcard_present(path):
703     return path.translate(None, "*#@%") != path
704
705 class Command:
706     def __init__(self):
707         self.usage = "usage: %prog [options]"
708         self.needsGit = True
709         self.verbose = False
710
711 class P4UserMap:
712     def __init__(self):
713         self.userMapFromPerforceServer = False
714         self.myP4UserId = None
715
716     def p4UserId(self):
717         if self.myP4UserId:
718             return self.myP4UserId
719
720         results = p4CmdList("user -o")
721         for r in results:
722             if r.has_key('User'):
723                 self.myP4UserId = r['User']
724                 return r['User']
725         die("Could not find your p4 user id")
726
727     def p4UserIsMe(self, p4User):
728         # return True if the given p4 user is actually me
729         me = self.p4UserId()
730         if not p4User or p4User != me:
731             return False
732         else:
733             return True
734
735     def getUserCacheFilename(self):
736         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
737         return home + "/.gitp4-usercache.txt"
738
739     def getUserMapFromPerforceServer(self):
740         if self.userMapFromPerforceServer:
741             return
742         self.users = {}
743         self.emails = {}
744
745         for output in p4CmdList("users"):
746             if not output.has_key("User"):
747                 continue
748             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
749             self.emails[output["Email"]] = output["User"]
750
751
752         s = ''
753         for (key, val) in self.users.items():
754             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
755
756         open(self.getUserCacheFilename(), "wb").write(s)
757         self.userMapFromPerforceServer = True
758
759     def loadUserMapFromCache(self):
760         self.users = {}
761         self.userMapFromPerforceServer = False
762         try:
763             cache = open(self.getUserCacheFilename(), "rb")
764             lines = cache.readlines()
765             cache.close()
766             for line in lines:
767                 entry = line.strip().split("\t")
768                 self.users[entry[0]] = entry[1]
769         except IOError:
770             self.getUserMapFromPerforceServer()
771
772 class P4Debug(Command):
773     def __init__(self):
774         Command.__init__(self)
775         self.options = []
776         self.description = "A tool to debug the output of p4 -G."
777         self.needsGit = False
778
779     def run(self, args):
780         j = 0
781         for output in p4CmdList(args):
782             print 'Element: %d' % j
783             j += 1
784             print output
785         return True
786
787 class P4RollBack(Command):
788     def __init__(self):
789         Command.__init__(self)
790         self.options = [
791             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
792         ]
793         self.description = "A tool to debug the multi-branch import. Don't use :)"
794         self.rollbackLocalBranches = False
795
796     def run(self, args):
797         if len(args) != 1:
798             return False
799         maxChange = int(args[0])
800
801         if "p4ExitCode" in p4Cmd("changes -m 1"):
802             die("Problems executing p4");
803
804         if self.rollbackLocalBranches:
805             refPrefix = "refs/heads/"
806             lines = read_pipe_lines("git rev-parse --symbolic --branches")
807         else:
808             refPrefix = "refs/remotes/"
809             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
810
811         for line in lines:
812             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
813                 line = line.strip()
814                 ref = refPrefix + line
815                 log = extractLogMessageFromGitCommit(ref)
816                 settings = extractSettingsGitLog(log)
817
818                 depotPaths = settings['depot-paths']
819                 change = settings['change']
820
821                 changed = False
822
823                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
824                                                            for p in depotPaths]))) == 0:
825                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
826                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
827                     continue
828
829                 while change and int(change) > maxChange:
830                     changed = True
831                     if self.verbose:
832                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
833                     system("git update-ref %s \"%s^\"" % (ref, ref))
834                     log = extractLogMessageFromGitCommit(ref)
835                     settings =  extractSettingsGitLog(log)
836
837
838                     depotPaths = settings['depot-paths']
839                     change = settings['change']
840
841                 if changed:
842                     print "%s rewound to %s" % (ref, change)
843
844         return True
845
846 class P4Submit(Command, P4UserMap):
847     def __init__(self):
848         Command.__init__(self)
849         P4UserMap.__init__(self)
850         self.options = [
851                 optparse.make_option("--origin", dest="origin"),
852                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
853                 # preserve the user, requires relevant p4 permissions
854                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
855                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
856         ]
857         self.description = "Submit changes from git to the perforce depot."
858         self.usage += " [name of git branch to submit into perforce depot]"
859         self.origin = ""
860         self.detectRenames = False
861         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
862         self.isWindows = (platform.system() == "Windows")
863         self.exportLabels = False
864         self.p4HasMoveCommand = p4_has_command("move")
865
866     def check(self):
867         if len(p4CmdList("opened ...")) > 0:
868             die("You have files opened with perforce! Close them before starting the sync.")
869
870     def separate_jobs_from_description(self, message):
871         """Extract and return a possible Jobs field in the commit
872            message.  It goes into a separate section in the p4 change
873            specification.
874
875            A jobs line starts with "Jobs:" and looks like a new field
876            in a form.  Values are white-space separated on the same
877            line or on following lines that start with a tab.
878
879            This does not parse and extract the full git commit message
880            like a p4 form.  It just sees the Jobs: line as a marker
881            to pass everything from then on directly into the p4 form,
882            but outside the description section.
883
884            Return a tuple (stripped log message, jobs string)."""
885
886         m = re.search(r'^Jobs:', message, re.MULTILINE)
887         if m is None:
888             return (message, None)
889
890         jobtext = message[m.start():]
891         stripped_message = message[:m.start()].rstrip()
892         return (stripped_message, jobtext)
893
894     def prepareLogMessage(self, template, message, jobs):
895         """Edits the template returned from "p4 change -o" to insert
896            the message in the Description field, and the jobs text in
897            the Jobs field."""
898         result = ""
899
900         inDescriptionSection = False
901
902         for line in template.split("\n"):
903             if line.startswith("#"):
904                 result += line + "\n"
905                 continue
906
907             if inDescriptionSection:
908                 if line.startswith("Files:") or line.startswith("Jobs:"):
909                     inDescriptionSection = False
910                     # insert Jobs section
911                     if jobs:
912                         result += jobs + "\n"
913                 else:
914                     continue
915             else:
916                 if line.startswith("Description:"):
917                     inDescriptionSection = True
918                     line += "\n"
919                     for messageLine in message.split("\n"):
920                         line += "\t" + messageLine + "\n"
921
922             result += line + "\n"
923
924         return result
925
926     def patchRCSKeywords(self, file, pattern):
927         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
928         (handle, outFileName) = tempfile.mkstemp(dir='.')
929         try:
930             outFile = os.fdopen(handle, "w+")
931             inFile = open(file, "r")
932             regexp = re.compile(pattern, re.VERBOSE)
933             for line in inFile.readlines():
934                 line = regexp.sub(r'$\1$', line)
935                 outFile.write(line)
936             inFile.close()
937             outFile.close()
938             # Forcibly overwrite the original file
939             os.unlink(file)
940             shutil.move(outFileName, file)
941         except:
942             # cleanup our temporary file
943             os.unlink(outFileName)
944             print "Failed to strip RCS keywords in %s" % file
945             raise
946
947         print "Patched up RCS keywords in %s" % file
948
949     def p4UserForCommit(self,id):
950         # Return the tuple (perforce user,git email) for a given git commit id
951         self.getUserMapFromPerforceServer()
952         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
953         gitEmail = gitEmail.strip()
954         if not self.emails.has_key(gitEmail):
955             return (None,gitEmail)
956         else:
957             return (self.emails[gitEmail],gitEmail)
958
959     def checkValidP4Users(self,commits):
960         # check if any git authors cannot be mapped to p4 users
961         for id in commits:
962             (user,email) = self.p4UserForCommit(id)
963             if not user:
964                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
965                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
966                     print "%s" % msg
967                 else:
968                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
969
970     def lastP4Changelist(self):
971         # Get back the last changelist number submitted in this client spec. This
972         # then gets used to patch up the username in the change. If the same
973         # client spec is being used by multiple processes then this might go
974         # wrong.
975         results = p4CmdList("client -o")        # find the current client
976         client = None
977         for r in results:
978             if r.has_key('Client'):
979                 client = r['Client']
980                 break
981         if not client:
982             die("could not get client spec")
983         results = p4CmdList(["changes", "-c", client, "-m", "1"])
984         for r in results:
985             if r.has_key('change'):
986                 return r['change']
987         die("Could not get changelist number for last submit - cannot patch up user details")
988
989     def modifyChangelistUser(self, changelist, newUser):
990         # fixup the user field of a changelist after it has been submitted.
991         changes = p4CmdList("change -o %s" % changelist)
992         if len(changes) != 1:
993             die("Bad output from p4 change modifying %s to user %s" %
994                 (changelist, newUser))
995
996         c = changes[0]
997         if c['User'] == newUser: return   # nothing to do
998         c['User'] = newUser
999         input = marshal.dumps(c)
1000
1001         result = p4CmdList("change -f -i", stdin=input)
1002         for r in result:
1003             if r.has_key('code'):
1004                 if r['code'] == 'error':
1005                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1006             if r.has_key('data'):
1007                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1008                 return
1009         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1010
1011     def canChangeChangelists(self):
1012         # check to see if we have p4 admin or super-user permissions, either of
1013         # which are required to modify changelists.
1014         results = p4CmdList(["protects", self.depotPath])
1015         for r in results:
1016             if r.has_key('perm'):
1017                 if r['perm'] == 'admin':
1018                     return 1
1019                 if r['perm'] == 'super':
1020                     return 1
1021         return 0
1022
1023     def prepareSubmitTemplate(self):
1024         """Run "p4 change -o" to grab a change specification template.
1025            This does not use "p4 -G", as it is nice to keep the submission
1026            template in original order, since a human might edit it.
1027
1028            Remove lines in the Files section that show changes to files
1029            outside the depot path we're committing into."""
1030
1031         template = ""
1032         inFilesSection = False
1033         for line in p4_read_pipe_lines(['change', '-o']):
1034             if line.endswith("\r\n"):
1035                 line = line[:-2] + "\n"
1036             if inFilesSection:
1037                 if line.startswith("\t"):
1038                     # path starts and ends with a tab
1039                     path = line[1:]
1040                     lastTab = path.rfind("\t")
1041                     if lastTab != -1:
1042                         path = path[:lastTab]
1043                         if not p4PathStartsWith(path, self.depotPath):
1044                             continue
1045                 else:
1046                     inFilesSection = False
1047             else:
1048                 if line.startswith("Files:"):
1049                     inFilesSection = True
1050
1051             template += line
1052
1053         return template
1054
1055     def edit_template(self, template_file):
1056         """Invoke the editor to let the user change the submission
1057            message.  Return true if okay to continue with the submit."""
1058
1059         # if configured to skip the editing part, just submit
1060         if gitConfig("git-p4.skipSubmitEdit") == "true":
1061             return True
1062
1063         # look at the modification time, to check later if the user saved
1064         # the file
1065         mtime = os.stat(template_file).st_mtime
1066
1067         # invoke the editor
1068         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1069             editor = os.environ.get("P4EDITOR")
1070         else:
1071             editor = read_pipe("git var GIT_EDITOR").strip()
1072         system(editor + " " + template_file)
1073
1074         # If the file was not saved, prompt to see if this patch should
1075         # be skipped.  But skip this verification step if configured so.
1076         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1077             return True
1078
1079         # modification time updated means user saved the file
1080         if os.stat(template_file).st_mtime > mtime:
1081             return True
1082
1083         while True:
1084             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1085             if response == 'y':
1086                 return True
1087             if response == 'n':
1088                 return False
1089
1090     def applyCommit(self, id):
1091         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1092
1093         (p4User, gitEmail) = self.p4UserForCommit(id)
1094
1095         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1096         filesToAdd = set()
1097         filesToDelete = set()
1098         editedFiles = set()
1099         pureRenameCopy = set()
1100         filesToChangeExecBit = {}
1101
1102         for line in diff:
1103             diff = parseDiffTreeEntry(line)
1104             modifier = diff['status']
1105             path = diff['src']
1106             if modifier == "M":
1107                 p4_edit(path)
1108                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1109                     filesToChangeExecBit[path] = diff['dst_mode']
1110                 editedFiles.add(path)
1111             elif modifier == "A":
1112                 filesToAdd.add(path)
1113                 filesToChangeExecBit[path] = diff['dst_mode']
1114                 if path in filesToDelete:
1115                     filesToDelete.remove(path)
1116             elif modifier == "D":
1117                 filesToDelete.add(path)
1118                 if path in filesToAdd:
1119                     filesToAdd.remove(path)
1120             elif modifier == "C":
1121                 src, dest = diff['src'], diff['dst']
1122                 p4_integrate(src, dest)
1123                 pureRenameCopy.add(dest)
1124                 if diff['src_sha1'] != diff['dst_sha1']:
1125                     p4_edit(dest)
1126                     pureRenameCopy.discard(dest)
1127                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1128                     p4_edit(dest)
1129                     pureRenameCopy.discard(dest)
1130                     filesToChangeExecBit[dest] = diff['dst_mode']
1131                 os.unlink(dest)
1132                 editedFiles.add(dest)
1133             elif modifier == "R":
1134                 src, dest = diff['src'], diff['dst']
1135                 if self.p4HasMoveCommand:
1136                     p4_edit(src)        # src must be open before move
1137                     p4_move(src, dest)  # opens for (move/delete, move/add)
1138                 else:
1139                     p4_integrate(src, dest)
1140                     if diff['src_sha1'] != diff['dst_sha1']:
1141                         p4_edit(dest)
1142                     else:
1143                         pureRenameCopy.add(dest)
1144                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1145                     if not self.p4HasMoveCommand:
1146                         p4_edit(dest)   # with move: already open, writable
1147                     filesToChangeExecBit[dest] = diff['dst_mode']
1148                 if not self.p4HasMoveCommand:
1149                     os.unlink(dest)
1150                     filesToDelete.add(src)
1151                 editedFiles.add(dest)
1152             else:
1153                 die("unknown modifier %s for %s" % (modifier, path))
1154
1155         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1156         patchcmd = diffcmd + " | git apply "
1157         tryPatchCmd = patchcmd + "--check -"
1158         applyPatchCmd = patchcmd + "--check --apply -"
1159         patch_succeeded = True
1160
1161         if os.system(tryPatchCmd) != 0:
1162             fixed_rcs_keywords = False
1163             patch_succeeded = False
1164             print "Unfortunately applying the change failed!"
1165
1166             # Patch failed, maybe it's just RCS keyword woes. Look through
1167             # the patch to see if that's possible.
1168             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1169                 file = None
1170                 pattern = None
1171                 kwfiles = {}
1172                 for file in editedFiles | filesToDelete:
1173                     # did this file's delta contain RCS keywords?
1174                     pattern = p4_keywords_regexp_for_file(file)
1175
1176                     if pattern:
1177                         # this file is a possibility...look for RCS keywords.
1178                         regexp = re.compile(pattern, re.VERBOSE)
1179                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1180                             if regexp.search(line):
1181                                 if verbose:
1182                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1183                                 kwfiles[file] = pattern
1184                                 break
1185
1186                 for file in kwfiles:
1187                     if verbose:
1188                         print "zapping %s with %s" % (line,pattern)
1189                     self.patchRCSKeywords(file, kwfiles[file])
1190                     fixed_rcs_keywords = True
1191
1192             if fixed_rcs_keywords:
1193                 print "Retrying the patch with RCS keywords cleaned up"
1194                 if os.system(tryPatchCmd) == 0:
1195                     patch_succeeded = True
1196
1197         if not patch_succeeded:
1198             print "What do you want to do?"
1199             response = "x"
1200             while response != "s" and response != "a" and response != "w":
1201                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1202                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1203             if response == "s":
1204                 print "Skipping! Good luck with the next patches..."
1205                 for f in editedFiles:
1206                     p4_revert(f)
1207                 for f in filesToAdd:
1208                     os.remove(f)
1209                 return
1210             elif response == "a":
1211                 os.system(applyPatchCmd)
1212                 if len(filesToAdd) > 0:
1213                     print "You may also want to call p4 add on the following files:"
1214                     print " ".join(filesToAdd)
1215                 if len(filesToDelete):
1216                     print "The following files should be scheduled for deletion with p4 delete:"
1217                     print " ".join(filesToDelete)
1218                 die("Please resolve and submit the conflict manually and "
1219                     + "continue afterwards with git p4 submit --continue")
1220             elif response == "w":
1221                 system(diffcmd + " > patch.txt")
1222                 print "Patch saved to patch.txt in %s !" % self.clientPath
1223                 die("Please resolve and submit the conflict manually and "
1224                     "continue afterwards with git p4 submit --continue")
1225
1226         system(applyPatchCmd)
1227
1228         for f in filesToAdd:
1229             p4_add(f)
1230         for f in filesToDelete:
1231             p4_revert(f)
1232             p4_delete(f)
1233
1234         # Set/clear executable bits
1235         for f in filesToChangeExecBit.keys():
1236             mode = filesToChangeExecBit[f]
1237             setP4ExecBit(f, mode)
1238
1239         logMessage = extractLogMessageFromGitCommit(id)
1240         logMessage = logMessage.strip()
1241         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1242
1243         template = self.prepareSubmitTemplate()
1244         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1245
1246         if self.preserveUser:
1247            submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1248
1249         if os.environ.has_key("P4DIFF"):
1250             del(os.environ["P4DIFF"])
1251         diff = ""
1252         for editedFile in editedFiles:
1253             diff += p4_read_pipe(['diff', '-du',
1254                                   wildcard_encode(editedFile)])
1255
1256         newdiff = ""
1257         for newFile in filesToAdd:
1258             newdiff += "==== new file ====\n"
1259             newdiff += "--- /dev/null\n"
1260             newdiff += "+++ %s\n" % newFile
1261             f = open(newFile, "r")
1262             for line in f.readlines():
1263                 newdiff += "+" + line
1264             f.close()
1265
1266         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1267             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1268             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1269             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1270
1271         separatorLine = "######## everything below this line is just the diff #######\n"
1272
1273         (handle, fileName) = tempfile.mkstemp()
1274         tmpFile = os.fdopen(handle, "w+")
1275         if self.isWindows:
1276             submitTemplate = submitTemplate.replace("\n", "\r\n")
1277             separatorLine = separatorLine.replace("\n", "\r\n")
1278             newdiff = newdiff.replace("\n", "\r\n")
1279         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1280         tmpFile.close()
1281
1282         if self.edit_template(fileName):
1283             # read the edited message and submit
1284             tmpFile = open(fileName, "rb")
1285             message = tmpFile.read()
1286             tmpFile.close()
1287             submitTemplate = message[:message.index(separatorLine)]
1288             if self.isWindows:
1289                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1290             p4_write_pipe(['submit', '-i'], submitTemplate)
1291
1292             if self.preserveUser:
1293                 if p4User:
1294                     # Get last changelist number. Cannot easily get it from
1295                     # the submit command output as the output is
1296                     # unmarshalled.
1297                     changelist = self.lastP4Changelist()
1298                     self.modifyChangelistUser(changelist, p4User)
1299
1300             # The rename/copy happened by applying a patch that created a
1301             # new file.  This leaves it writable, which confuses p4.
1302             for f in pureRenameCopy:
1303                 p4_sync(f, "-f")
1304
1305         else:
1306             # skip this patch
1307             print "Submission cancelled, undoing p4 changes."
1308             for f in editedFiles:
1309                 p4_revert(f)
1310             for f in filesToAdd:
1311                 p4_revert(f)
1312                 os.remove(f)
1313
1314         os.remove(fileName)
1315
1316     # Export git tags as p4 labels. Create a p4 label and then tag
1317     # with that.
1318     def exportGitTags(self, gitTags):
1319         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1320         if len(validLabelRegexp) == 0:
1321             validLabelRegexp = defaultLabelRegexp
1322         m = re.compile(validLabelRegexp)
1323
1324         for name in gitTags:
1325
1326             if not m.match(name):
1327                 if verbose:
1328                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1329                 continue
1330
1331             # Get the p4 commit this corresponds to
1332             logMessage = extractLogMessageFromGitCommit(name)
1333             values = extractSettingsGitLog(logMessage)
1334
1335             if not values.has_key('change'):
1336                 # a tag pointing to something not sent to p4; ignore
1337                 if verbose:
1338                     print "git tag %s does not give a p4 commit" % name
1339                 continue
1340             else:
1341                 changelist = values['change']
1342
1343             # Get the tag details.
1344             inHeader = True
1345             isAnnotated = False
1346             body = []
1347             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1348                 l = l.strip()
1349                 if inHeader:
1350                     if re.match(r'tag\s+', l):
1351                         isAnnotated = True
1352                     elif re.match(r'\s*$', l):
1353                         inHeader = False
1354                         continue
1355                 else:
1356                     body.append(l)
1357
1358             if not isAnnotated:
1359                 body = ["lightweight tag imported by git p4\n"]
1360
1361             # Create the label - use the same view as the client spec we are using
1362             clientSpec = getClientSpec()
1363
1364             labelTemplate  = "Label: %s\n" % name
1365             labelTemplate += "Description:\n"
1366             for b in body:
1367                 labelTemplate += "\t" + b + "\n"
1368             labelTemplate += "View:\n"
1369             for mapping in clientSpec.mappings:
1370                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1371
1372             p4_write_pipe(["label", "-i"], labelTemplate)
1373
1374             # Use the label
1375             p4_system(["tag", "-l", name] +
1376                       ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1377
1378             if verbose:
1379                 print "created p4 label for tag %s" % name
1380
1381     def run(self, args):
1382         if len(args) == 0:
1383             self.master = currentGitBranch()
1384             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1385                 die("Detecting current git branch failed!")
1386         elif len(args) == 1:
1387             self.master = args[0]
1388             if not branchExists(self.master):
1389                 die("Branch %s does not exist" % self.master)
1390         else:
1391             return False
1392
1393         allowSubmit = gitConfig("git-p4.allowSubmit")
1394         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1395             die("%s is not in git-p4.allowSubmit" % self.master)
1396
1397         [upstream, settings] = findUpstreamBranchPoint()
1398         self.depotPath = settings['depot-paths'][0]
1399         if len(self.origin) == 0:
1400             self.origin = upstream
1401
1402         if self.preserveUser:
1403             if not self.canChangeChangelists():
1404                 die("Cannot preserve user names without p4 super-user or admin permissions")
1405
1406         if self.verbose:
1407             print "Origin branch is " + self.origin
1408
1409         if len(self.depotPath) == 0:
1410             print "Internal error: cannot locate perforce depot path from existing branches"
1411             sys.exit(128)
1412
1413         self.useClientSpec = False
1414         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1415             self.useClientSpec = True
1416         if self.useClientSpec:
1417             self.clientSpecDirs = getClientSpec()
1418
1419         if self.useClientSpec:
1420             # all files are relative to the client spec
1421             self.clientPath = getClientRoot()
1422         else:
1423             self.clientPath = p4Where(self.depotPath)
1424
1425         if self.clientPath == "":
1426             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1427
1428         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1429         self.oldWorkingDirectory = os.getcwd()
1430
1431         # ensure the clientPath exists
1432         new_client_dir = False
1433         if not os.path.exists(self.clientPath):
1434             new_client_dir = True
1435             os.makedirs(self.clientPath)
1436
1437         chdir(self.clientPath)
1438         print "Synchronizing p4 checkout..."
1439         if new_client_dir:
1440             # old one was destroyed, and maybe nobody told p4
1441             p4_sync("...", "-f")
1442         else:
1443             p4_sync("...")
1444         self.check()
1445
1446         commits = []
1447         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1448             commits.append(line.strip())
1449         commits.reverse()
1450
1451         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1452             self.checkAuthorship = False
1453         else:
1454             self.checkAuthorship = True
1455
1456         if self.preserveUser:
1457             self.checkValidP4Users(commits)
1458
1459         #
1460         # Build up a set of options to be passed to diff when
1461         # submitting each commit to p4.
1462         #
1463         if self.detectRenames:
1464             # command-line -M arg
1465             self.diffOpts = "-M"
1466         else:
1467             # If not explicitly set check the config variable
1468             detectRenames = gitConfig("git-p4.detectRenames")
1469
1470             if detectRenames.lower() == "false" or detectRenames == "":
1471                 self.diffOpts = ""
1472             elif detectRenames.lower() == "true":
1473                 self.diffOpts = "-M"
1474             else:
1475                 self.diffOpts = "-M%s" % detectRenames
1476
1477         # no command-line arg for -C or --find-copies-harder, just
1478         # config variables
1479         detectCopies = gitConfig("git-p4.detectCopies")
1480         if detectCopies.lower() == "false" or detectCopies == "":
1481             pass
1482         elif detectCopies.lower() == "true":
1483             self.diffOpts += " -C"
1484         else:
1485             self.diffOpts += " -C%s" % detectCopies
1486
1487         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1488             self.diffOpts += " --find-copies-harder"
1489
1490         while len(commits) > 0:
1491             commit = commits[0]
1492             commits = commits[1:]
1493             self.applyCommit(commit)
1494
1495         if len(commits) == 0:
1496             print "All changes applied!"
1497             chdir(self.oldWorkingDirectory)
1498
1499             sync = P4Sync()
1500             sync.run([])
1501
1502             rebase = P4Rebase()
1503             rebase.rebase()
1504
1505         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1506             self.exportLabels = True
1507
1508         if self.exportLabels:
1509             p4Labels = getP4Labels(self.depotPath)
1510             gitTags = getGitTags()
1511
1512             missingGitTags = gitTags - p4Labels
1513             self.exportGitTags(missingGitTags)
1514
1515         return True
1516
1517 class View(object):
1518     """Represent a p4 view ("p4 help views"), and map files in a
1519        repo according to the view."""
1520
1521     class Path(object):
1522         """A depot or client path, possibly containing wildcards.
1523            The only one supported is ... at the end, currently.
1524            Initialize with the full path, with //depot or //client."""
1525
1526         def __init__(self, path, is_depot):
1527             self.path = path
1528             self.is_depot = is_depot
1529             self.find_wildcards()
1530             # remember the prefix bit, useful for relative mappings
1531             m = re.match("(//[^/]+/)", self.path)
1532             if not m:
1533                 die("Path %s does not start with //prefix/" % self.path)
1534             prefix = m.group(1)
1535             if not self.is_depot:
1536                 # strip //client/ on client paths
1537                 self.path = self.path[len(prefix):]
1538
1539         def find_wildcards(self):
1540             """Make sure wildcards are valid, and set up internal
1541                variables."""
1542
1543             self.ends_triple_dot = False
1544             # There are three wildcards allowed in p4 views
1545             # (see "p4 help views").  This code knows how to
1546             # handle "..." (only at the end), but cannot deal with
1547             # "%%n" or "*".  Only check the depot_side, as p4 should
1548             # validate that the client_side matches too.
1549             if re.search(r'%%[1-9]', self.path):
1550                 die("Can't handle %%n wildcards in view: %s" % self.path)
1551             if self.path.find("*") >= 0:
1552                 die("Can't handle * wildcards in view: %s" % self.path)
1553             triple_dot_index = self.path.find("...")
1554             if triple_dot_index >= 0:
1555                 if triple_dot_index != len(self.path) - 3:
1556                     die("Can handle only single ... wildcard, at end: %s" %
1557                         self.path)
1558                 self.ends_triple_dot = True
1559
1560         def ensure_compatible(self, other_path):
1561             """Make sure the wildcards agree."""
1562             if self.ends_triple_dot != other_path.ends_triple_dot:
1563                  die("Both paths must end with ... if either does;\n" +
1564                      "paths: %s %s" % (self.path, other_path.path))
1565
1566         def match_wildcards(self, test_path):
1567             """See if this test_path matches us, and fill in the value
1568                of the wildcards if so.  Returns a tuple of
1569                (True|False, wildcards[]).  For now, only the ... at end
1570                is supported, so at most one wildcard."""
1571             if self.ends_triple_dot:
1572                 dotless = self.path[:-3]
1573                 if test_path.startswith(dotless):
1574                     wildcard = test_path[len(dotless):]
1575                     return (True, [ wildcard ])
1576             else:
1577                 if test_path == self.path:
1578                     return (True, [])
1579             return (False, [])
1580
1581         def match(self, test_path):
1582             """Just return if it matches; don't bother with the wildcards."""
1583             b, _ = self.match_wildcards(test_path)
1584             return b
1585
1586         def fill_in_wildcards(self, wildcards):
1587             """Return the relative path, with the wildcards filled in
1588                if there are any."""
1589             if self.ends_triple_dot:
1590                 return self.path[:-3] + wildcards[0]
1591             else:
1592                 return self.path
1593
1594     class Mapping(object):
1595         def __init__(self, depot_side, client_side, overlay, exclude):
1596             # depot_side is without the trailing /... if it had one
1597             self.depot_side = View.Path(depot_side, is_depot=True)
1598             self.client_side = View.Path(client_side, is_depot=False)
1599             self.overlay = overlay  # started with "+"
1600             self.exclude = exclude  # started with "-"
1601             assert not (self.overlay and self.exclude)
1602             self.depot_side.ensure_compatible(self.client_side)
1603
1604         def __str__(self):
1605             c = " "
1606             if self.overlay:
1607                 c = "+"
1608             if self.exclude:
1609                 c = "-"
1610             return "View.Mapping: %s%s -> %s" % \
1611                    (c, self.depot_side.path, self.client_side.path)
1612
1613         def map_depot_to_client(self, depot_path):
1614             """Calculate the client path if using this mapping on the
1615                given depot path; does not consider the effect of other
1616                mappings in a view.  Even excluded mappings are returned."""
1617             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1618             if not matches:
1619                 return ""
1620             client_path = self.client_side.fill_in_wildcards(wildcards)
1621             return client_path
1622
1623     #
1624     # View methods
1625     #
1626     def __init__(self):
1627         self.mappings = []
1628
1629     def append(self, view_line):
1630         """Parse a view line, splitting it into depot and client
1631            sides.  Append to self.mappings, preserving order."""
1632
1633         # Split the view line into exactly two words.  P4 enforces
1634         # structure on these lines that simplifies this quite a bit.
1635         #
1636         # Either or both words may be double-quoted.
1637         # Single quotes do not matter.
1638         # Double-quote marks cannot occur inside the words.
1639         # A + or - prefix is also inside the quotes.
1640         # There are no quotes unless they contain a space.
1641         # The line is already white-space stripped.
1642         # The two words are separated by a single space.
1643         #
1644         if view_line[0] == '"':
1645             # First word is double quoted.  Find its end.
1646             close_quote_index = view_line.find('"', 1)
1647             if close_quote_index <= 0:
1648                 die("No first-word closing quote found: %s" % view_line)
1649             depot_side = view_line[1:close_quote_index]
1650             # skip closing quote and space
1651             rhs_index = close_quote_index + 1 + 1
1652         else:
1653             space_index = view_line.find(" ")
1654             if space_index <= 0:
1655                 die("No word-splitting space found: %s" % view_line)
1656             depot_side = view_line[0:space_index]
1657             rhs_index = space_index + 1
1658
1659         if view_line[rhs_index] == '"':
1660             # Second word is double quoted.  Make sure there is a
1661             # double quote at the end too.
1662             if not view_line.endswith('"'):
1663                 die("View line with rhs quote should end with one: %s" %
1664                     view_line)
1665             # skip the quotes
1666             client_side = view_line[rhs_index+1:-1]
1667         else:
1668             client_side = view_line[rhs_index:]
1669
1670         # prefix + means overlay on previous mapping
1671         overlay = False
1672         if depot_side.startswith("+"):
1673             overlay = True
1674             depot_side = depot_side[1:]
1675
1676         # prefix - means exclude this path
1677         exclude = False
1678         if depot_side.startswith("-"):
1679             exclude = True
1680             depot_side = depot_side[1:]
1681
1682         m = View.Mapping(depot_side, client_side, overlay, exclude)
1683         self.mappings.append(m)
1684
1685     def map_in_client(self, depot_path):
1686         """Return the relative location in the client where this
1687            depot file should live.  Returns "" if the file should
1688            not be mapped in the client."""
1689
1690         paths_filled = []
1691         client_path = ""
1692
1693         # look at later entries first
1694         for m in self.mappings[::-1]:
1695
1696             # see where will this path end up in the client
1697             p = m.map_depot_to_client(depot_path)
1698
1699             if p == "":
1700                 # Depot path does not belong in client.  Must remember
1701                 # this, as previous items should not cause files to
1702                 # exist in this path either.  Remember that the list is
1703                 # being walked from the end, which has higher precedence.
1704                 # Overlap mappings do not exclude previous mappings.
1705                 if not m.overlay:
1706                     paths_filled.append(m.client_side)
1707
1708             else:
1709                 # This mapping matched; no need to search any further.
1710                 # But, the mapping could be rejected if the client path
1711                 # has already been claimed by an earlier mapping (i.e.
1712                 # one later in the list, which we are walking backwards).
1713                 already_mapped_in_client = False
1714                 for f in paths_filled:
1715                     # this is View.Path.match
1716                     if f.match(p):
1717                         already_mapped_in_client = True
1718                         break
1719                 if not already_mapped_in_client:
1720                     # Include this file, unless it is from a line that
1721                     # explicitly said to exclude it.
1722                     if not m.exclude:
1723                         client_path = p
1724
1725                 # a match, even if rejected, always stops the search
1726                 break
1727
1728         return client_path
1729
1730 class P4Sync(Command, P4UserMap):
1731     delete_actions = ( "delete", "move/delete", "purge" )
1732
1733     def __init__(self):
1734         Command.__init__(self)
1735         P4UserMap.__init__(self)
1736         self.options = [
1737                 optparse.make_option("--branch", dest="branch"),
1738                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1739                 optparse.make_option("--changesfile", dest="changesFile"),
1740                 optparse.make_option("--silent", dest="silent", action="store_true"),
1741                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1742                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1743                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1744                                      help="Import into refs/heads/ , not refs/remotes"),
1745                 optparse.make_option("--max-changes", dest="maxChanges"),
1746                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1747                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1748                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1749                                      help="Only sync files that are included in the Perforce Client Spec")
1750         ]
1751         self.description = """Imports from Perforce into a git repository.\n
1752     example:
1753     //depot/my/project/ -- to import the current head
1754     //depot/my/project/@all -- to import everything
1755     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1756
1757     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1758
1759         self.usage += " //depot/path[@revRange]"
1760         self.silent = False
1761         self.createdBranches = set()
1762         self.committedChanges = set()
1763         self.branch = ""
1764         self.detectBranches = False
1765         self.detectLabels = False
1766         self.importLabels = False
1767         self.changesFile = ""
1768         self.syncWithOrigin = True
1769         self.importIntoRemotes = True
1770         self.maxChanges = ""
1771         self.isWindows = (platform.system() == "Windows")
1772         self.keepRepoPath = False
1773         self.depotPaths = None
1774         self.p4BranchesInGit = []
1775         self.cloneExclude = []
1776         self.useClientSpec = False
1777         self.useClientSpec_from_options = False
1778         self.clientSpecDirs = None
1779         self.tempBranches = []
1780         self.tempBranchLocation = "git-p4-tmp"
1781
1782         if gitConfig("git-p4.syncFromOrigin") == "false":
1783             self.syncWithOrigin = False
1784
1785     # Force a checkpoint in fast-import and wait for it to finish
1786     def checkpoint(self):
1787         self.gitStream.write("checkpoint\n\n")
1788         self.gitStream.write("progress checkpoint\n\n")
1789         out = self.gitOutput.readline()
1790         if self.verbose:
1791             print "checkpoint finished: " + out
1792
1793     def extractFilesFromCommit(self, commit):
1794         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1795                              for path in self.cloneExclude]
1796         files = []
1797         fnum = 0
1798         while commit.has_key("depotFile%s" % fnum):
1799             path =  commit["depotFile%s" % fnum]
1800
1801             if [p for p in self.cloneExclude
1802                 if p4PathStartsWith(path, p)]:
1803                 found = False
1804             else:
1805                 found = [p for p in self.depotPaths
1806                          if p4PathStartsWith(path, p)]
1807             if not found:
1808                 fnum = fnum + 1
1809                 continue
1810
1811             file = {}
1812             file["path"] = path
1813             file["rev"] = commit["rev%s" % fnum]
1814             file["action"] = commit["action%s" % fnum]
1815             file["type"] = commit["type%s" % fnum]
1816             files.append(file)
1817             fnum = fnum + 1
1818         return files
1819
1820     def stripRepoPath(self, path, prefixes):
1821         if self.useClientSpec:
1822             return self.clientSpecDirs.map_in_client(path)
1823
1824         if self.keepRepoPath:
1825             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1826
1827         for p in prefixes:
1828             if p4PathStartsWith(path, p):
1829                 path = path[len(p):]
1830
1831         return path
1832
1833     def splitFilesIntoBranches(self, commit):
1834         branches = {}
1835         fnum = 0
1836         while commit.has_key("depotFile%s" % fnum):
1837             path =  commit["depotFile%s" % fnum]
1838             found = [p for p in self.depotPaths
1839                      if p4PathStartsWith(path, p)]
1840             if not found:
1841                 fnum = fnum + 1
1842                 continue
1843
1844             file = {}
1845             file["path"] = path
1846             file["rev"] = commit["rev%s" % fnum]
1847             file["action"] = commit["action%s" % fnum]
1848             file["type"] = commit["type%s" % fnum]
1849             fnum = fnum + 1
1850
1851             relPath = self.stripRepoPath(path, self.depotPaths)
1852             relPath = wildcard_decode(relPath)
1853
1854             for branch in self.knownBranches.keys():
1855
1856                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1857                 if relPath.startswith(branch + "/"):
1858                     if branch not in branches:
1859                         branches[branch] = []
1860                     branches[branch].append(file)
1861                     break
1862
1863         return branches
1864
1865     # output one file from the P4 stream
1866     # - helper for streamP4Files
1867
1868     def streamOneP4File(self, file, contents):
1869         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1870         relPath = wildcard_decode(relPath)
1871         if verbose:
1872             sys.stderr.write("%s\n" % relPath)
1873
1874         (type_base, type_mods) = split_p4_type(file["type"])
1875
1876         git_mode = "100644"
1877         if "x" in type_mods:
1878             git_mode = "100755"
1879         if type_base == "symlink":
1880             git_mode = "120000"
1881             # p4 print on a symlink contains "target\n"; remove the newline
1882             data = ''.join(contents)
1883             contents = [data[:-1]]
1884
1885         if type_base == "utf16":
1886             # p4 delivers different text in the python output to -G
1887             # than it does when using "print -o", or normal p4 client
1888             # operations.  utf16 is converted to ascii or utf8, perhaps.
1889             # But ascii text saved as -t utf16 is completely mangled.
1890             # Invoke print -o to get the real contents.
1891             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1892             contents = [ text ]
1893
1894         if type_base == "apple":
1895             # Apple filetype files will be streamed as a concatenation of
1896             # its appledouble header and the contents.  This is useless
1897             # on both macs and non-macs.  If using "print -q -o xx", it
1898             # will create "xx" with the data, and "%xx" with the header.
1899             # This is also not very useful.
1900             #
1901             # Ideally, someday, this script can learn how to generate
1902             # appledouble files directly and import those to git, but
1903             # non-mac machines can never find a use for apple filetype.
1904             print "\nIgnoring apple filetype file %s" % file['depotFile']
1905             return
1906
1907         # Perhaps windows wants unicode, utf16 newlines translated too;
1908         # but this is not doing it.
1909         if self.isWindows and type_base == "text":
1910             mangled = []
1911             for data in contents:
1912                 data = data.replace("\r\n", "\n")
1913                 mangled.append(data)
1914             contents = mangled
1915
1916         # Note that we do not try to de-mangle keywords on utf16 files,
1917         # even though in theory somebody may want that.
1918         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1919         if pattern:
1920             regexp = re.compile(pattern, re.VERBOSE)
1921             text = ''.join(contents)
1922             text = regexp.sub(r'$\1$', text)
1923             contents = [ text ]
1924
1925         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1926
1927         # total length...
1928         length = 0
1929         for d in contents:
1930             length = length + len(d)
1931
1932         self.gitStream.write("data %d\n" % length)
1933         for d in contents:
1934             self.gitStream.write(d)
1935         self.gitStream.write("\n")
1936
1937     def streamOneP4Deletion(self, file):
1938         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1939         relPath = wildcard_decode(relPath)
1940         if verbose:
1941             sys.stderr.write("delete %s\n" % relPath)
1942         self.gitStream.write("D %s\n" % relPath)
1943
1944     # handle another chunk of streaming data
1945     def streamP4FilesCb(self, marshalled):
1946
1947         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1948             # start of a new file - output the old one first
1949             self.streamOneP4File(self.stream_file, self.stream_contents)
1950             self.stream_file = {}
1951             self.stream_contents = []
1952             self.stream_have_file_info = False
1953
1954         # pick up the new file information... for the
1955         # 'data' field we need to append to our array
1956         for k in marshalled.keys():
1957             if k == 'data':
1958                 self.stream_contents.append(marshalled['data'])
1959             else:
1960                 self.stream_file[k] = marshalled[k]
1961
1962         self.stream_have_file_info = True
1963
1964     # Stream directly from "p4 files" into "git fast-import"
1965     def streamP4Files(self, files):
1966         filesForCommit = []
1967         filesToRead = []
1968         filesToDelete = []
1969
1970         for f in files:
1971             # if using a client spec, only add the files that have
1972             # a path in the client
1973             if self.clientSpecDirs:
1974                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1975                     continue
1976
1977             filesForCommit.append(f)
1978             if f['action'] in self.delete_actions:
1979                 filesToDelete.append(f)
1980             else:
1981                 filesToRead.append(f)
1982
1983         # deleted files...
1984         for f in filesToDelete:
1985             self.streamOneP4Deletion(f)
1986
1987         if len(filesToRead) > 0:
1988             self.stream_file = {}
1989             self.stream_contents = []
1990             self.stream_have_file_info = False
1991
1992             # curry self argument
1993             def streamP4FilesCbSelf(entry):
1994                 self.streamP4FilesCb(entry)
1995
1996             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1997
1998             p4CmdList(["-x", "-", "print"],
1999                       stdin=fileArgs,
2000                       cb=streamP4FilesCbSelf)
2001
2002             # do the last chunk
2003             if self.stream_file.has_key('depotFile'):
2004                 self.streamOneP4File(self.stream_file, self.stream_contents)
2005
2006     def make_email(self, userid):
2007         if userid in self.users:
2008             return self.users[userid]
2009         else:
2010             return "%s <a@b>" % userid
2011
2012     # Stream a p4 tag
2013     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2014         if verbose:
2015             print "writing tag %s for commit %s" % (labelName, commit)
2016         gitStream.write("tag %s\n" % labelName)
2017         gitStream.write("from %s\n" % commit)
2018
2019         if labelDetails.has_key('Owner'):
2020             owner = labelDetails["Owner"]
2021         else:
2022             owner = None
2023
2024         # Try to use the owner of the p4 label, or failing that,
2025         # the current p4 user id.
2026         if owner:
2027             email = self.make_email(owner)
2028         else:
2029             email = self.make_email(self.p4UserId())
2030         tagger = "%s %s %s" % (email, epoch, self.tz)
2031
2032         gitStream.write("tagger %s\n" % tagger)
2033
2034         print "labelDetails=",labelDetails
2035         if labelDetails.has_key('Description'):
2036             description = labelDetails['Description']
2037         else:
2038             description = 'Label from git p4'
2039
2040         gitStream.write("data %d\n" % len(description))
2041         gitStream.write(description)
2042         gitStream.write("\n")
2043
2044     def commit(self, details, files, branch, branchPrefixes, parent = ""):
2045         epoch = details["time"]
2046         author = details["user"]
2047         self.branchPrefixes = branchPrefixes
2048
2049         if self.verbose:
2050             print "commit into %s" % branch
2051
2052         # start with reading files; if that fails, we should not
2053         # create a commit.
2054         new_files = []
2055         for f in files:
2056             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
2057                 new_files.append (f)
2058             else:
2059                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2060
2061         self.gitStream.write("commit %s\n" % branch)
2062 #        gitStream.write("mark :%s\n" % details["change"])
2063         self.committedChanges.add(int(details["change"]))
2064         committer = ""
2065         if author not in self.users:
2066             self.getUserMapFromPerforceServer()
2067         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2068
2069         self.gitStream.write("committer %s\n" % committer)
2070
2071         self.gitStream.write("data <<EOT\n")
2072         self.gitStream.write(details["desc"])
2073         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2074                              % (','.join (branchPrefixes), details["change"]))
2075         if len(details['options']) > 0:
2076             self.gitStream.write(": options = %s" % details['options'])
2077         self.gitStream.write("]\nEOT\n\n")
2078
2079         if len(parent) > 0:
2080             if self.verbose:
2081                 print "parent %s" % parent
2082             self.gitStream.write("from %s\n" % parent)
2083
2084         self.streamP4Files(new_files)
2085         self.gitStream.write("\n")
2086
2087         change = int(details["change"])
2088
2089         if self.labels.has_key(change):
2090             label = self.labels[change]
2091             labelDetails = label[0]
2092             labelRevisions = label[1]
2093             if self.verbose:
2094                 print "Change %s is labelled %s" % (change, labelDetails)
2095
2096             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2097                                                     for p in branchPrefixes])
2098
2099             if len(files) == len(labelRevisions):
2100
2101                 cleanedFiles = {}
2102                 for info in files:
2103                     if info["action"] in self.delete_actions:
2104                         continue
2105                     cleanedFiles[info["depotFile"]] = info["rev"]
2106
2107                 if cleanedFiles == labelRevisions:
2108                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2109
2110                 else:
2111                     if not self.silent:
2112                         print ("Tag %s does not match with change %s: files do not match."
2113                                % (labelDetails["label"], change))
2114
2115             else:
2116                 if not self.silent:
2117                     print ("Tag %s does not match with change %s: file count is different."
2118                            % (labelDetails["label"], change))
2119
2120     # Build a dictionary of changelists and labels, for "detect-labels" option.
2121     def getLabels(self):
2122         self.labels = {}
2123
2124         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2125         if len(l) > 0 and not self.silent:
2126             print "Finding files belonging to labels in %s" % `self.depotPaths`
2127
2128         for output in l:
2129             label = output["label"]
2130             revisions = {}
2131             newestChange = 0
2132             if self.verbose:
2133                 print "Querying files for label %s" % label
2134             for file in p4CmdList(["files"] +
2135                                       ["%s...@%s" % (p, label)
2136                                           for p in self.depotPaths]):
2137                 revisions[file["depotFile"]] = file["rev"]
2138                 change = int(file["change"])
2139                 if change > newestChange:
2140                     newestChange = change
2141
2142             self.labels[newestChange] = [output, revisions]
2143
2144         if self.verbose:
2145             print "Label changes: %s" % self.labels.keys()
2146
2147     # Import p4 labels as git tags. A direct mapping does not
2148     # exist, so assume that if all the files are at the same revision
2149     # then we can use that, or it's something more complicated we should
2150     # just ignore.
2151     def importP4Labels(self, stream, p4Labels):
2152         if verbose:
2153             print "import p4 labels: " + ' '.join(p4Labels)
2154
2155         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2156         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2157         if len(validLabelRegexp) == 0:
2158             validLabelRegexp = defaultLabelRegexp
2159         m = re.compile(validLabelRegexp)
2160
2161         for name in p4Labels:
2162             commitFound = False
2163
2164             if not m.match(name):
2165                 if verbose:
2166                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2167                 continue
2168
2169             if name in ignoredP4Labels:
2170                 continue
2171
2172             labelDetails = p4CmdList(['label', "-o", name])[0]
2173
2174             # get the most recent changelist for each file in this label
2175             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2176                                 for p in self.depotPaths])
2177
2178             if change.has_key('change'):
2179                 # find the corresponding git commit; take the oldest commit
2180                 changelist = int(change['change'])
2181                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2182                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2183                 if len(gitCommit) == 0:
2184                     print "could not find git commit for changelist %d" % changelist
2185                 else:
2186                     gitCommit = gitCommit.strip()
2187                     commitFound = True
2188                     # Convert from p4 time format
2189                     try:
2190                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2191                     except ValueError:
2192                         print "Could not convert label time %s" % labelDetail['Update']
2193                         tmwhen = 1
2194
2195                     when = int(time.mktime(tmwhen))
2196                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2197                     if verbose:
2198                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2199             else:
2200                 if verbose:
2201                     print "Label %s has no changelists - possibly deleted?" % name
2202
2203             if not commitFound:
2204                 # We can't import this label; don't try again as it will get very
2205                 # expensive repeatedly fetching all the files for labels that will
2206                 # never be imported. If the label is moved in the future, the
2207                 # ignore will need to be removed manually.
2208                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2209
2210     def guessProjectName(self):
2211         for p in self.depotPaths:
2212             if p.endswith("/"):
2213                 p = p[:-1]
2214             p = p[p.strip().rfind("/") + 1:]
2215             if not p.endswith("/"):
2216                p += "/"
2217             return p
2218
2219     def getBranchMapping(self):
2220         lostAndFoundBranches = set()
2221
2222         user = gitConfig("git-p4.branchUser")
2223         if len(user) > 0:
2224             command = "branches -u %s" % user
2225         else:
2226             command = "branches"
2227
2228         for info in p4CmdList(command):
2229             details = p4Cmd(["branch", "-o", info["branch"]])
2230             viewIdx = 0
2231             while details.has_key("View%s" % viewIdx):
2232                 paths = details["View%s" % viewIdx].split(" ")
2233                 viewIdx = viewIdx + 1
2234                 # require standard //depot/foo/... //depot/bar/... mapping
2235                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2236                     continue
2237                 source = paths[0]
2238                 destination = paths[1]
2239                 ## HACK
2240                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2241                     source = source[len(self.depotPaths[0]):-4]
2242                     destination = destination[len(self.depotPaths[0]):-4]
2243
2244                     if destination in self.knownBranches:
2245                         if not self.silent:
2246                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2247                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2248                         continue
2249
2250                     self.knownBranches[destination] = source
2251
2252                     lostAndFoundBranches.discard(destination)
2253
2254                     if source not in self.knownBranches:
2255                         lostAndFoundBranches.add(source)
2256
2257         # Perforce does not strictly require branches to be defined, so we also
2258         # check git config for a branch list.
2259         #
2260         # Example of branch definition in git config file:
2261         # [git-p4]
2262         #   branchList=main:branchA
2263         #   branchList=main:branchB
2264         #   branchList=branchA:branchC
2265         configBranches = gitConfigList("git-p4.branchList")
2266         for branch in configBranches:
2267             if branch:
2268                 (source, destination) = branch.split(":")
2269                 self.knownBranches[destination] = source
2270
2271                 lostAndFoundBranches.discard(destination)
2272
2273                 if source not in self.knownBranches:
2274                     lostAndFoundBranches.add(source)
2275
2276
2277         for branch in lostAndFoundBranches:
2278             self.knownBranches[branch] = branch
2279
2280     def getBranchMappingFromGitBranches(self):
2281         branches = p4BranchesInGit(self.importIntoRemotes)
2282         for branch in branches.keys():
2283             if branch == "master":
2284                 branch = "main"
2285             else:
2286                 branch = branch[len(self.projectName):]
2287             self.knownBranches[branch] = branch
2288
2289     def listExistingP4GitBranches(self):
2290         # branches holds mapping from name to commit
2291         branches = p4BranchesInGit(self.importIntoRemotes)
2292         self.p4BranchesInGit = branches.keys()
2293         for branch in branches.keys():
2294             self.initialParents[self.refPrefix + branch] = branches[branch]
2295
2296     def updateOptionDict(self, d):
2297         option_keys = {}
2298         if self.keepRepoPath:
2299             option_keys['keepRepoPath'] = 1
2300
2301         d["options"] = ' '.join(sorted(option_keys.keys()))
2302
2303     def readOptions(self, d):
2304         self.keepRepoPath = (d.has_key('options')
2305                              and ('keepRepoPath' in d['options']))
2306
2307     def gitRefForBranch(self, branch):
2308         if branch == "main":
2309             return self.refPrefix + "master"
2310
2311         if len(branch) <= 0:
2312             return branch
2313
2314         return self.refPrefix + self.projectName + branch
2315
2316     def gitCommitByP4Change(self, ref, change):
2317         if self.verbose:
2318             print "looking in ref " + ref + " for change %s using bisect..." % change
2319
2320         earliestCommit = ""
2321         latestCommit = parseRevision(ref)
2322
2323         while True:
2324             if self.verbose:
2325                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2326             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2327             if len(next) == 0:
2328                 if self.verbose:
2329                     print "argh"
2330                 return ""
2331             log = extractLogMessageFromGitCommit(next)
2332             settings = extractSettingsGitLog(log)
2333             currentChange = int(settings['change'])
2334             if self.verbose:
2335                 print "current change %s" % currentChange
2336
2337             if currentChange == change:
2338                 if self.verbose:
2339                     print "found %s" % next
2340                 return next
2341
2342             if currentChange < change:
2343                 earliestCommit = "^%s" % next
2344             else:
2345                 latestCommit = "%s" % next
2346
2347         return ""
2348
2349     def importNewBranch(self, branch, maxChange):
2350         # make fast-import flush all changes to disk and update the refs using the checkpoint
2351         # command so that we can try to find the branch parent in the git history
2352         self.gitStream.write("checkpoint\n\n");
2353         self.gitStream.flush();
2354         branchPrefix = self.depotPaths[0] + branch + "/"
2355         range = "@1,%s" % maxChange
2356         #print "prefix" + branchPrefix
2357         changes = p4ChangesForPaths([branchPrefix], range)
2358         if len(changes) <= 0:
2359             return False
2360         firstChange = changes[0]
2361         #print "first change in branch: %s" % firstChange
2362         sourceBranch = self.knownBranches[branch]
2363         sourceDepotPath = self.depotPaths[0] + sourceBranch
2364         sourceRef = self.gitRefForBranch(sourceBranch)
2365         #print "source " + sourceBranch
2366
2367         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2368         #print "branch parent: %s" % branchParentChange
2369         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2370         if len(gitParent) > 0:
2371             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2372             #print "parent git commit: %s" % gitParent
2373
2374         self.importChanges(changes)
2375         return True
2376
2377     def searchParent(self, parent, branch, target):
2378         parentFound = False
2379         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2380             blob = blob.strip()
2381             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2382                 parentFound = True
2383                 if self.verbose:
2384                     print "Found parent of %s in commit %s" % (branch, blob)
2385                 break
2386         if parentFound:
2387             return blob
2388         else:
2389             return None
2390
2391     def importChanges(self, changes):
2392         cnt = 1
2393         for change in changes:
2394             description = p4Cmd(["describe", str(change)])
2395             self.updateOptionDict(description)
2396
2397             if not self.silent:
2398                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2399                 sys.stdout.flush()
2400             cnt = cnt + 1
2401
2402             try:
2403                 if self.detectBranches:
2404                     branches = self.splitFilesIntoBranches(description)
2405                     for branch in branches.keys():
2406                         ## HACK  --hwn
2407                         branchPrefix = self.depotPaths[0] + branch + "/"
2408
2409                         parent = ""
2410
2411                         filesForCommit = branches[branch]
2412
2413                         if self.verbose:
2414                             print "branch is %s" % branch
2415
2416                         self.updatedBranches.add(branch)
2417
2418                         if branch not in self.createdBranches:
2419                             self.createdBranches.add(branch)
2420                             parent = self.knownBranches[branch]
2421                             if parent == branch:
2422                                 parent = ""
2423                             else:
2424                                 fullBranch = self.projectName + branch
2425                                 if fullBranch not in self.p4BranchesInGit:
2426                                     if not self.silent:
2427                                         print("\n    Importing new branch %s" % fullBranch);
2428                                     if self.importNewBranch(branch, change - 1):
2429                                         parent = ""
2430                                         self.p4BranchesInGit.append(fullBranch)
2431                                     if not self.silent:
2432                                         print("\n    Resuming with change %s" % change);
2433
2434                                 if self.verbose:
2435                                     print "parent determined through known branches: %s" % parent
2436
2437                         branch = self.gitRefForBranch(branch)
2438                         parent = self.gitRefForBranch(parent)
2439
2440                         if self.verbose:
2441                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2442
2443                         if len(parent) == 0 and branch in self.initialParents:
2444                             parent = self.initialParents[branch]
2445                             del self.initialParents[branch]
2446
2447                         blob = None
2448                         if len(parent) > 0:
2449                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2450                             if self.verbose:
2451                                 print "Creating temporary branch: " + tempBranch
2452                             self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2453                             self.tempBranches.append(tempBranch)
2454                             self.checkpoint()
2455                             blob = self.searchParent(parent, branch, tempBranch)
2456                         if blob:
2457                             self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2458                         else:
2459                             if self.verbose:
2460                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2461                             self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2462                 else:
2463                     files = self.extractFilesFromCommit(description)
2464                     self.commit(description, files, self.branch, self.depotPaths,
2465                                 self.initialParent)
2466                     self.initialParent = ""
2467             except IOError:
2468                 print self.gitError.read()
2469                 sys.exit(1)
2470
2471     def importHeadRevision(self, revision):
2472         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2473
2474         details = {}
2475         details["user"] = "git perforce import user"
2476         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2477                            % (' '.join(self.depotPaths), revision))
2478         details["change"] = revision
2479         newestRevision = 0
2480
2481         fileCnt = 0
2482         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2483
2484         for info in p4CmdList(["files"] + fileArgs):
2485
2486             if 'code' in info and info['code'] == 'error':
2487                 sys.stderr.write("p4 returned an error: %s\n"
2488                                  % info['data'])
2489                 if info['data'].find("must refer to client") >= 0:
2490                     sys.stderr.write("This particular p4 error is misleading.\n")
2491                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2492                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2493                 sys.exit(1)
2494             if 'p4ExitCode' in info:
2495                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2496                 sys.exit(1)
2497
2498
2499             change = int(info["change"])
2500             if change > newestRevision:
2501                 newestRevision = change
2502
2503             if info["action"] in self.delete_actions:
2504                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2505                 #fileCnt = fileCnt + 1
2506                 continue
2507
2508             for prop in ["depotFile", "rev", "action", "type" ]:
2509                 details["%s%s" % (prop, fileCnt)] = info[prop]
2510
2511             fileCnt = fileCnt + 1
2512
2513         details["change"] = newestRevision
2514
2515         # Use time from top-most change so that all git p4 clones of
2516         # the same p4 repo have the same commit SHA1s.
2517         res = p4CmdList("describe -s %d" % newestRevision)
2518         newestTime = None
2519         for r in res:
2520             if r.has_key('time'):
2521                 newestTime = int(r['time'])
2522         if newestTime is None:
2523             die("\"describe -s\" on newest change %d did not give a time")
2524         details["time"] = newestTime
2525
2526         self.updateOptionDict(details)
2527         try:
2528             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2529         except IOError:
2530             print "IO error with git fast-import. Is your git version recent enough?"
2531             print self.gitError.read()
2532
2533
2534     def run(self, args):
2535         self.depotPaths = []
2536         self.changeRange = ""
2537         self.initialParent = ""
2538         self.previousDepotPaths = []
2539
2540         # map from branch depot path to parent branch
2541         self.knownBranches = {}
2542         self.initialParents = {}
2543         self.hasOrigin = originP4BranchesExist()
2544         if not self.syncWithOrigin:
2545             self.hasOrigin = False
2546
2547         if self.importIntoRemotes:
2548             self.refPrefix = "refs/remotes/p4/"
2549         else:
2550             self.refPrefix = "refs/heads/p4/"
2551
2552         if self.syncWithOrigin and self.hasOrigin:
2553             if not self.silent:
2554                 print "Syncing with origin first by calling git fetch origin"
2555             system("git fetch origin")
2556
2557         if len(self.branch) == 0:
2558             self.branch = self.refPrefix + "master"
2559             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2560                 system("git update-ref %s refs/heads/p4" % self.branch)
2561                 system("git branch -D p4");
2562             # create it /after/ importing, when master exists
2563             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2564                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2565
2566         # accept either the command-line option, or the configuration variable
2567         if self.useClientSpec:
2568             # will use this after clone to set the variable
2569             self.useClientSpec_from_options = True
2570         else:
2571             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2572                 self.useClientSpec = True
2573         if self.useClientSpec:
2574             self.clientSpecDirs = getClientSpec()
2575
2576         # TODO: should always look at previous commits,
2577         # merge with previous imports, if possible.
2578         if args == []:
2579             if self.hasOrigin:
2580                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2581             self.listExistingP4GitBranches()
2582
2583             if len(self.p4BranchesInGit) > 1:
2584                 if not self.silent:
2585                     print "Importing from/into multiple branches"
2586                 self.detectBranches = True
2587
2588             if self.verbose:
2589                 print "branches: %s" % self.p4BranchesInGit
2590
2591             p4Change = 0
2592             for branch in self.p4BranchesInGit:
2593                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2594
2595                 settings = extractSettingsGitLog(logMsg)
2596
2597                 self.readOptions(settings)
2598                 if (settings.has_key('depot-paths')
2599                     and settings.has_key ('change')):
2600                     change = int(settings['change']) + 1
2601                     p4Change = max(p4Change, change)
2602
2603                     depotPaths = sorted(settings['depot-paths'])
2604                     if self.previousDepotPaths == []:
2605                         self.previousDepotPaths = depotPaths
2606                     else:
2607                         paths = []
2608                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2609                             prev_list = prev.split("/")
2610                             cur_list = cur.split("/")
2611                             for i in range(0, min(len(cur_list), len(prev_list))):
2612                                 if cur_list[i] <> prev_list[i]:
2613                                     i = i - 1
2614                                     break
2615
2616                             paths.append ("/".join(cur_list[:i + 1]))
2617
2618                         self.previousDepotPaths = paths
2619
2620             if p4Change > 0:
2621                 self.depotPaths = sorted(self.previousDepotPaths)
2622                 self.changeRange = "@%s,#head" % p4Change
2623                 if not self.detectBranches:
2624                     self.initialParent = parseRevision(self.branch)
2625                 if not self.silent and not self.detectBranches:
2626                     print "Performing incremental import into %s git branch" % self.branch
2627
2628         if not self.branch.startswith("refs/"):
2629             self.branch = "refs/heads/" + self.branch
2630
2631         if len(args) == 0 and self.depotPaths:
2632             if not self.silent:
2633                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2634         else:
2635             if self.depotPaths and self.depotPaths != args:
2636                 print ("previous import used depot path %s and now %s was specified. "
2637                        "This doesn't work!" % (' '.join (self.depotPaths),
2638                                                ' '.join (args)))
2639                 sys.exit(1)
2640
2641             self.depotPaths = sorted(args)
2642
2643         revision = ""
2644         self.users = {}
2645
2646         # Make sure no revision specifiers are used when --changesfile
2647         # is specified.
2648         bad_changesfile = False
2649         if len(self.changesFile) > 0:
2650             for p in self.depotPaths:
2651                 if p.find("@") >= 0 or p.find("#") >= 0:
2652                     bad_changesfile = True
2653                     break
2654         if bad_changesfile:
2655             die("Option --changesfile is incompatible with revision specifiers")
2656
2657         newPaths = []
2658         for p in self.depotPaths:
2659             if p.find("@") != -1:
2660                 atIdx = p.index("@")
2661                 self.changeRange = p[atIdx:]
2662                 if self.changeRange == "@all":
2663                     self.changeRange = ""
2664                 elif ',' not in self.changeRange:
2665                     revision = self.changeRange
2666                     self.changeRange = ""
2667                 p = p[:atIdx]
2668             elif p.find("#") != -1:
2669                 hashIdx = p.index("#")
2670                 revision = p[hashIdx:]
2671                 p = p[:hashIdx]
2672             elif self.previousDepotPaths == []:
2673                 # pay attention to changesfile, if given, else import
2674                 # the entire p4 tree at the head revision
2675                 if len(self.changesFile) == 0:
2676                     revision = "#head"
2677
2678             p = re.sub ("\.\.\.$", "", p)
2679             if not p.endswith("/"):
2680                 p += "/"
2681
2682             newPaths.append(p)
2683
2684         self.depotPaths = newPaths
2685
2686         self.loadUserMapFromCache()
2687         self.labels = {}
2688         if self.detectLabels:
2689             self.getLabels();
2690
2691         if self.detectBranches:
2692             ## FIXME - what's a P4 projectName ?
2693             self.projectName = self.guessProjectName()
2694
2695             if self.hasOrigin:
2696                 self.getBranchMappingFromGitBranches()
2697             else:
2698                 self.getBranchMapping()
2699             if self.verbose:
2700                 print "p4-git branches: %s" % self.p4BranchesInGit
2701                 print "initial parents: %s" % self.initialParents
2702             for b in self.p4BranchesInGit:
2703                 if b != "master":
2704
2705                     ## FIXME
2706                     b = b[len(self.projectName):]
2707                 self.createdBranches.add(b)
2708
2709         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2710
2711         importProcess = subprocess.Popen(["git", "fast-import"],
2712                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2713                                          stderr=subprocess.PIPE);
2714         self.gitOutput = importProcess.stdout
2715         self.gitStream = importProcess.stdin
2716         self.gitError = importProcess.stderr
2717
2718         if revision:
2719             self.importHeadRevision(revision)
2720         else:
2721             changes = []
2722
2723             if len(self.changesFile) > 0:
2724                 output = open(self.changesFile).readlines()
2725                 changeSet = set()
2726                 for line in output:
2727                     changeSet.add(int(line))
2728
2729                 for change in changeSet:
2730                     changes.append(change)
2731
2732                 changes.sort()
2733             else:
2734                 # catch "git p4 sync" with no new branches, in a repo that
2735                 # does not have any existing p4 branches
2736                 if len(args) == 0 and not self.p4BranchesInGit:
2737                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2738                 if self.verbose:
2739                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2740                                                               self.changeRange)
2741                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2742
2743                 if len(self.maxChanges) > 0:
2744                     changes = changes[:min(int(self.maxChanges), len(changes))]
2745
2746             if len(changes) == 0:
2747                 if not self.silent:
2748                     print "No changes to import!"
2749             else:
2750                 if not self.silent and not self.detectBranches:
2751                     print "Import destination: %s" % self.branch
2752
2753                 self.updatedBranches = set()
2754
2755                 self.importChanges(changes)
2756
2757                 if not self.silent:
2758                     print ""
2759                     if len(self.updatedBranches) > 0:
2760                         sys.stdout.write("Updated branches: ")
2761                         for b in self.updatedBranches:
2762                             sys.stdout.write("%s " % b)
2763                         sys.stdout.write("\n")
2764
2765         if gitConfig("git-p4.importLabels", "--bool") == "true":
2766             self.importLabels = True
2767
2768         if self.importLabels:
2769             p4Labels = getP4Labels(self.depotPaths)
2770             gitTags = getGitTags()
2771
2772             missingP4Labels = p4Labels - gitTags
2773             self.importP4Labels(self.gitStream, missingP4Labels)
2774
2775         self.gitStream.close()
2776         if importProcess.wait() != 0:
2777             die("fast-import failed: %s" % self.gitError.read())
2778         self.gitOutput.close()
2779         self.gitError.close()
2780
2781         # Cleanup temporary branches created during import
2782         if self.tempBranches != []:
2783             for branch in self.tempBranches:
2784                 read_pipe("git update-ref -d %s" % branch)
2785             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2786
2787         return True
2788
2789 class P4Rebase(Command):
2790     def __init__(self):
2791         Command.__init__(self)
2792         self.options = [
2793                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2794         ]
2795         self.importLabels = False
2796         self.description = ("Fetches the latest revision from perforce and "
2797                             + "rebases the current work (branch) against it")
2798
2799     def run(self, args):
2800         sync = P4Sync()
2801         sync.importLabels = self.importLabels
2802         sync.run([])
2803
2804         return self.rebase()
2805
2806     def rebase(self):
2807         if os.system("git update-index --refresh") != 0:
2808             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2809         if len(read_pipe("git diff-index HEAD --")) > 0:
2810             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2811
2812         [upstream, settings] = findUpstreamBranchPoint()
2813         if len(upstream) == 0:
2814             die("Cannot find upstream branchpoint for rebase")
2815
2816         # the branchpoint may be p4/foo~3, so strip off the parent
2817         upstream = re.sub("~[0-9]+$", "", upstream)
2818
2819         print "Rebasing the current branch onto %s" % upstream
2820         oldHead = read_pipe("git rev-parse HEAD").strip()
2821         system("git rebase %s" % upstream)
2822         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2823         return True
2824
2825 class P4Clone(P4Sync):
2826     def __init__(self):
2827         P4Sync.__init__(self)
2828         self.description = "Creates a new git repository and imports from Perforce into it"
2829         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2830         self.options += [
2831             optparse.make_option("--destination", dest="cloneDestination",
2832                                  action='store', default=None,
2833                                  help="where to leave result of the clone"),
2834             optparse.make_option("-/", dest="cloneExclude",
2835                                  action="append", type="string",
2836                                  help="exclude depot path"),
2837             optparse.make_option("--bare", dest="cloneBare",
2838                                  action="store_true", default=False),
2839         ]
2840         self.cloneDestination = None
2841         self.needsGit = False
2842         self.cloneBare = False
2843
2844     # This is required for the "append" cloneExclude action
2845     def ensure_value(self, attr, value):
2846         if not hasattr(self, attr) or getattr(self, attr) is None:
2847             setattr(self, attr, value)
2848         return getattr(self, attr)
2849
2850     def defaultDestination(self, args):
2851         ## TODO: use common prefix of args?
2852         depotPath = args[0]
2853         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2854         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2855         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2856         depotDir = re.sub(r"/$", "", depotDir)
2857         return os.path.split(depotDir)[1]
2858
2859     def run(self, args):
2860         if len(args) < 1:
2861             return False
2862
2863         if self.keepRepoPath and not self.cloneDestination:
2864             sys.stderr.write("Must specify destination for --keep-path\n")
2865             sys.exit(1)
2866
2867         depotPaths = args
2868
2869         if not self.cloneDestination and len(depotPaths) > 1:
2870             self.cloneDestination = depotPaths[-1]
2871             depotPaths = depotPaths[:-1]
2872
2873         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2874         for p in depotPaths:
2875             if not p.startswith("//"):
2876                 return False
2877
2878         if not self.cloneDestination:
2879             self.cloneDestination = self.defaultDestination(args)
2880
2881         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2882
2883         if not os.path.exists(self.cloneDestination):
2884             os.makedirs(self.cloneDestination)
2885         chdir(self.cloneDestination)
2886
2887         init_cmd = [ "git", "init" ]
2888         if self.cloneBare:
2889             init_cmd.append("--bare")
2890         subprocess.check_call(init_cmd)
2891
2892         if not P4Sync.run(self, depotPaths):
2893             return False
2894         if self.branch != "master":
2895             if self.importIntoRemotes:
2896                 masterbranch = "refs/remotes/p4/master"
2897             else:
2898                 masterbranch = "refs/heads/p4/master"
2899             if gitBranchExists(masterbranch):
2900                 system("git branch master %s" % masterbranch)
2901                 if not self.cloneBare:
2902                     system("git checkout -f")
2903             else:
2904                 print "Could not detect main branch. No checkout/master branch created."
2905
2906         # auto-set this variable if invoked with --use-client-spec
2907         if self.useClientSpec_from_options:
2908             system("git config --bool git-p4.useclientspec true")
2909
2910         return True
2911
2912 class P4Branches(Command):
2913     def __init__(self):
2914         Command.__init__(self)
2915         self.options = [ ]
2916         self.description = ("Shows the git branches that hold imports and their "
2917                             + "corresponding perforce depot paths")
2918         self.verbose = False
2919
2920     def run(self, args):
2921         if originP4BranchesExist():
2922             createOrUpdateBranchesFromOrigin()
2923
2924         cmdline = "git rev-parse --symbolic "
2925         cmdline += " --remotes"
2926
2927         for line in read_pipe_lines(cmdline):
2928             line = line.strip()
2929
2930             if not line.startswith('p4/') or line == "p4/HEAD":
2931                 continue
2932             branch = line
2933
2934             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2935             settings = extractSettingsGitLog(log)
2936
2937             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2938         return True
2939
2940 class HelpFormatter(optparse.IndentedHelpFormatter):
2941     def __init__(self):
2942         optparse.IndentedHelpFormatter.__init__(self)
2943
2944     def format_description(self, description):
2945         if description:
2946             return description + "\n"
2947         else:
2948             return ""
2949
2950 def printUsage(commands):
2951     print "usage: %s <command> [options]" % sys.argv[0]
2952     print ""
2953     print "valid commands: %s" % ", ".join(commands)
2954     print ""
2955     print "Try %s <command> --help for command specific help." % sys.argv[0]
2956     print ""
2957
2958 commands = {
2959     "debug" : P4Debug,
2960     "submit" : P4Submit,
2961     "commit" : P4Submit,
2962     "sync" : P4Sync,
2963     "rebase" : P4Rebase,
2964     "clone" : P4Clone,
2965     "rollback" : P4RollBack,
2966     "branches" : P4Branches
2967 }
2968
2969
2970 def main():
2971     if len(sys.argv[1:]) == 0:
2972         printUsage(commands.keys())
2973         sys.exit(2)
2974
2975     cmd = ""
2976     cmdName = sys.argv[1]
2977     try:
2978         klass = commands[cmdName]
2979         cmd = klass()
2980     except KeyError:
2981         print "unknown command %s" % cmdName
2982         print ""
2983         printUsage(commands.keys())
2984         sys.exit(2)
2985
2986     options = cmd.options
2987     cmd.gitdir = os.environ.get("GIT_DIR", None)
2988
2989     args = sys.argv[2:]
2990
2991     options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))
2992     if cmd.needsGit:
2993         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2994
2995     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2996                                    options,
2997                                    description = cmd.description,
2998                                    formatter = HelpFormatter())
2999
3000     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3001     global verbose
3002     verbose = cmd.verbose
3003     if cmd.needsGit:
3004         if cmd.gitdir == None:
3005             cmd.gitdir = os.path.abspath(".git")
3006             if not isValidGitDir(cmd.gitdir):
3007                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3008                 if os.path.exists(cmd.gitdir):
3009                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3010                     if len(cdup) > 0:
3011                         chdir(cdup);
3012
3013         if not isValidGitDir(cmd.gitdir):
3014             if isValidGitDir(cmd.gitdir + "/.git"):
3015                 cmd.gitdir += "/.git"
3016             else:
3017                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3018
3019         os.environ["GIT_DIR"] = cmd.gitdir
3020
3021     if not cmd.run(args):
3022         parser.print_help()
3023         sys.exit(2)
3024
3025
3026 if __name__ == '__main__':
3027     main()