Merge branch 'maint'
[git] / contrib / fast-import / git-p4
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10
11 import optparse, sys, os, marshal, popen2, subprocess, shelve
12 import tempfile, getopt, sha, os.path, time, platform
13 import re
14
15 from sets import Set;
16
17 verbose = False
18
19 def die(msg):
20     if verbose:
21         raise Exception(msg)
22     else:
23         sys.stderr.write(msg + "\n")
24         sys.exit(1)
25
26 def write_pipe(c, str):
27     if verbose:
28         sys.stderr.write('Writing pipe: %s\n' % c)
29
30     pipe = os.popen(c, 'w')
31     val = pipe.write(str)
32     if pipe.close():
33         die('Command failed: %s' % c)
34
35     return val
36
37 def read_pipe(c, ignore_error=False):
38     if verbose:
39         sys.stderr.write('Reading pipe: %s\n' % c)
40
41     pipe = os.popen(c, 'rb')
42     val = pipe.read()
43     if pipe.close() and not ignore_error:
44         die('Command failed: %s' % c)
45
46     return val
47
48
49 def read_pipe_lines(c):
50     if verbose:
51         sys.stderr.write('Reading pipe: %s\n' % c)
52     ## todo: check return status
53     pipe = os.popen(c, 'rb')
54     val = pipe.readlines()
55     if pipe.close():
56         die('Command failed: %s' % c)
57
58     return val
59
60 def system(cmd):
61     if verbose:
62         sys.stderr.write("executing %s\n" % cmd)
63     if os.system(cmd) != 0:
64         die("command failed: %s" % cmd)
65
66 def isP4Exec(kind):
67     """Determine if a Perforce 'kind' should have execute permission
68
69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
71     a plus sign, it is also executable"""
72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
73
74 def setP4ExecBit(file, mode):
75     # Reopens an already open file and changes the execute bit to match
76     # the execute bit setting in the passed in mode.
77
78     p4Type = "+x"
79
80     if not isModeExec(mode):
81         p4Type = getP4OpenedType(file)
82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
84         if p4Type[-1] == "+":
85             p4Type = p4Type[0:-1]
86
87     system("p4 reopen -t %s %s" % (p4Type, file))
88
89 def getP4OpenedType(file):
90     # Returns the perforce file type for the given file.
91
92     result = read_pipe("p4 opened %s" % file)
93     match = re.match(".*\((.+)\)$", result)
94     if match:
95         return match.group(1)
96     else:
97         die("Could not determine file type for %s" % file)
98
99 def diffTreePattern():
100     # This is a simple generator for the diff tree regex pattern. This could be
101     # a class variable if this and parseDiffTreeEntry were a part of a class.
102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
103     while True:
104         yield pattern
105
106 def parseDiffTreeEntry(entry):
107     """Parses a single diff tree entry into its component elements.
108
109     See git-diff-tree(1) manpage for details about the format of the diff
110     output. This method returns a dictionary with the following elements:
111
112     src_mode - The mode of the source file
113     dst_mode - The mode of the destination file
114     src_sha1 - The sha1 for the source file
115     dst_sha1 - The sha1 fr the destination file
116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
117     status_score - The score for the status (applicable for 'C' and 'R'
118                    statuses). This is None if there is no score.
119     src - The path for the source file.
120     dst - The path for the destination file. This is only present for
121           copy or renames. If it is not present, this is None.
122
123     If the pattern is not matched, None is returned."""
124
125     match = diffTreePattern().next().match(entry)
126     if match:
127         return {
128             'src_mode': match.group(1),
129             'dst_mode': match.group(2),
130             'src_sha1': match.group(3),
131             'dst_sha1': match.group(4),
132             'status': match.group(5),
133             'status_score': match.group(6),
134             'src': match.group(7),
135             'dst': match.group(10)
136         }
137     return None
138
139 def isModeExec(mode):
140     # Returns True if the given git mode represents an executable file,
141     # otherwise False.
142     return mode[-3:] == "755"
143
144 def isModeExecChanged(src_mode, dst_mode):
145     return isModeExec(src_mode) != isModeExec(dst_mode)
146
147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
148     cmd = "p4 -G %s" % cmd
149     if verbose:
150         sys.stderr.write("Opening pipe: %s\n" % cmd)
151
152     # Use a temporary file to avoid deadlocks without
153     # subprocess.communicate(), which would put another copy
154     # of stdout into memory.
155     stdin_file = None
156     if stdin is not None:
157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
158         stdin_file.write(stdin)
159         stdin_file.flush()
160         stdin_file.seek(0)
161
162     p4 = subprocess.Popen(cmd, shell=True,
163                           stdin=stdin_file,
164                           stdout=subprocess.PIPE)
165
166     result = []
167     try:
168         while True:
169             entry = marshal.load(p4.stdout)
170             result.append(entry)
171     except EOFError:
172         pass
173     exitCode = p4.wait()
174     if exitCode != 0:
175         entry = {}
176         entry["p4ExitCode"] = exitCode
177         result.append(entry)
178
179     return result
180
181 def p4Cmd(cmd):
182     list = p4CmdList(cmd)
183     result = {}
184     for entry in list:
185         result.update(entry)
186     return result;
187
188 def p4Where(depotPath):
189     if not depotPath.endswith("/"):
190         depotPath += "/"
191     output = p4Cmd("where %s..." % depotPath)
192     if output["code"] == "error":
193         return ""
194     clientPath = ""
195     if "path" in output:
196         clientPath = output.get("path")
197     elif "data" in output:
198         data = output.get("data")
199         lastSpace = data.rfind(" ")
200         clientPath = data[lastSpace + 1:]
201
202     if clientPath.endswith("..."):
203         clientPath = clientPath[:-3]
204     return clientPath
205
206 def currentGitBranch():
207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
208
209 def isValidGitDir(path):
210     if (os.path.exists(path + "/HEAD")
211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
212         return True;
213     return False
214
215 def parseRevision(ref):
216     return read_pipe("git rev-parse %s" % ref).strip()
217
218 def extractLogMessageFromGitCommit(commit):
219     logMessage = ""
220
221     ## fixme: title is first line of commit, not 1st paragraph.
222     foundTitle = False
223     for log in read_pipe_lines("git cat-file commit %s" % commit):
224        if not foundTitle:
225            if len(log) == 1:
226                foundTitle = True
227            continue
228
229        logMessage += log
230     return logMessage
231
232 def extractSettingsGitLog(log):
233     values = {}
234     for line in log.split("\n"):
235         line = line.strip()
236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
237         if not m:
238             continue
239
240         assignments = m.group(1).split (':')
241         for a in assignments:
242             vals = a.split ('=')
243             key = vals[0].strip()
244             val = ('='.join (vals[1:])).strip()
245             if val.endswith ('\"') and val.startswith('"'):
246                 val = val[1:-1]
247
248             values[key] = val
249
250     paths = values.get("depot-paths")
251     if not paths:
252         paths = values.get("depot-path")
253     if paths:
254         values['depot-paths'] = paths.split(',')
255     return values
256
257 def gitBranchExists(branch):
258     proc = subprocess.Popen(["git", "rev-parse", branch],
259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
260     return proc.wait() == 0;
261
262 def gitConfig(key):
263     return read_pipe("git config %s" % key, ignore_error=True).strip()
264
265 def p4BranchesInGit(branchesAreInRemotes = True):
266     branches = {}
267
268     cmdline = "git rev-parse --symbolic "
269     if branchesAreInRemotes:
270         cmdline += " --remotes"
271     else:
272         cmdline += " --branches"
273
274     for line in read_pipe_lines(cmdline):
275         line = line.strip()
276
277         ## only import to p4/
278         if not line.startswith('p4/') or line == "p4/HEAD":
279             continue
280         branch = line
281
282         # strip off p4
283         branch = re.sub ("^p4/", "", line)
284
285         branches[branch] = parseRevision(line)
286     return branches
287
288 def findUpstreamBranchPoint(head = "HEAD"):
289     branches = p4BranchesInGit()
290     # map from depot-path to branch name
291     branchByDepotPath = {}
292     for branch in branches.keys():
293         tip = branches[branch]
294         log = extractLogMessageFromGitCommit(tip)
295         settings = extractSettingsGitLog(log)
296         if settings.has_key("depot-paths"):
297             paths = ",".join(settings["depot-paths"])
298             branchByDepotPath[paths] = "remotes/p4/" + branch
299
300     settings = None
301     parent = 0
302     while parent < 65535:
303         commit = head + "~%s" % parent
304         log = extractLogMessageFromGitCommit(commit)
305         settings = extractSettingsGitLog(log)
306         if settings.has_key("depot-paths"):
307             paths = ",".join(settings["depot-paths"])
308             if branchByDepotPath.has_key(paths):
309                 return [branchByDepotPath[paths], settings]
310
311         parent = parent + 1
312
313     return ["", settings]
314
315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
316     if not silent:
317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
318                % localRefPrefix)
319
320     originPrefix = "origin/p4/"
321
322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
323         line = line.strip()
324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
325             continue
326
327         headName = line[len(originPrefix):]
328         remoteHead = localRefPrefix + headName
329         originHead = line
330
331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
332         if (not original.has_key('depot-paths')
333             or not original.has_key('change')):
334             continue
335
336         update = False
337         if not gitBranchExists(remoteHead):
338             if verbose:
339                 print "creating %s" % remoteHead
340             update = True
341         else:
342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
343             if settings.has_key('change') > 0:
344                 if settings['depot-paths'] == original['depot-paths']:
345                     originP4Change = int(original['change'])
346                     p4Change = int(settings['change'])
347                     if originP4Change > p4Change:
348                         print ("%s (%s) is newer than %s (%s). "
349                                "Updating p4 branch from origin."
350                                % (originHead, originP4Change,
351                                   remoteHead, p4Change))
352                         update = True
353                 else:
354                     print ("Ignoring: %s was imported from %s while "
355                            "%s was imported from %s"
356                            % (originHead, ','.join(original['depot-paths']),
357                               remoteHead, ','.join(settings['depot-paths'])))
358
359         if update:
360             system("git update-ref %s %s" % (remoteHead, originHead))
361
362 def originP4BranchesExist():
363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
364
365 def p4ChangesForPaths(depotPaths, changeRange):
366     assert depotPaths
367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
368                                                         for p in depotPaths]))
369
370     changes = []
371     for line in output:
372         changeNum = line.split(" ")[1]
373         changes.append(int(changeNum))
374
375     changes.sort()
376     return changes
377
378 class Command:
379     def __init__(self):
380         self.usage = "usage: %prog [options]"
381         self.needsGit = True
382
383 class P4Debug(Command):
384     def __init__(self):
385         Command.__init__(self)
386         self.options = [
387             optparse.make_option("--verbose", dest="verbose", action="store_true",
388                                  default=False),
389             ]
390         self.description = "A tool to debug the output of p4 -G."
391         self.needsGit = False
392         self.verbose = False
393
394     def run(self, args):
395         j = 0
396         for output in p4CmdList(" ".join(args)):
397             print 'Element: %d' % j
398             j += 1
399             print output
400         return True
401
402 class P4RollBack(Command):
403     def __init__(self):
404         Command.__init__(self)
405         self.options = [
406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
408         ]
409         self.description = "A tool to debug the multi-branch import. Don't use :)"
410         self.verbose = False
411         self.rollbackLocalBranches = False
412
413     def run(self, args):
414         if len(args) != 1:
415             return False
416         maxChange = int(args[0])
417
418         if "p4ExitCode" in p4Cmd("changes -m 1"):
419             die("Problems executing p4");
420
421         if self.rollbackLocalBranches:
422             refPrefix = "refs/heads/"
423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
424         else:
425             refPrefix = "refs/remotes/"
426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
427
428         for line in lines:
429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
430                 line = line.strip()
431                 ref = refPrefix + line
432                 log = extractLogMessageFromGitCommit(ref)
433                 settings = extractSettingsGitLog(log)
434
435                 depotPaths = settings['depot-paths']
436                 change = settings['change']
437
438                 changed = False
439
440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
441                                                            for p in depotPaths]))) == 0:
442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
444                     continue
445
446                 while change and int(change) > maxChange:
447                     changed = True
448                     if self.verbose:
449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
450                     system("git update-ref %s \"%s^\"" % (ref, ref))
451                     log = extractLogMessageFromGitCommit(ref)
452                     settings =  extractSettingsGitLog(log)
453
454
455                     depotPaths = settings['depot-paths']
456                     change = settings['change']
457
458                 if changed:
459                     print "%s rewound to %s" % (ref, change)
460
461         return True
462
463 class P4Submit(Command):
464     def __init__(self):
465         Command.__init__(self)
466         self.options = [
467                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
468                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
469                 optparse.make_option("--origin", dest="origin"),
470                 optparse.make_option("--reset", action="store_true", dest="reset"),
471                 optparse.make_option("--log-substitutions", dest="substFile"),
472                 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
473                 optparse.make_option("-M", dest="detectRename", action="store_true"),
474         ]
475         self.description = "Submit changes from git to the perforce depot."
476         self.usage += " [name of git branch to submit into perforce depot]"
477         self.firstTime = True
478         self.reset = False
479         self.interactive = True
480         self.substFile = ""
481         self.firstTime = True
482         self.origin = ""
483         self.directSubmit = False
484         self.detectRename = False
485         self.verbose = False
486         self.isWindows = (platform.system() == "Windows")
487
488         self.logSubstitutions = {}
489         self.logSubstitutions["<enter description here>"] = "%log%"
490         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
491
492     def check(self):
493         if len(p4CmdList("opened ...")) > 0:
494             die("You have files opened with perforce! Close them before starting the sync.")
495
496     def start(self):
497         if len(self.config) > 0 and not self.reset:
498             die("Cannot start sync. Previous sync config found at %s\n"
499                 "If you want to start submitting again from scratch "
500                 "maybe you want to call git-p4 submit --reset" % self.configFile)
501
502         commits = []
503         if self.directSubmit:
504             commits.append("0")
505         else:
506             for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
507                 commits.append(line.strip())
508             commits.reverse()
509
510         self.config["commits"] = commits
511
512     def prepareLogMessage(self, template, message):
513         result = ""
514
515         for line in template.split("\n"):
516             if line.startswith("#"):
517                 result += line + "\n"
518                 continue
519
520             substituted = False
521             for key in self.logSubstitutions.keys():
522                 if line.find(key) != -1:
523                     value = self.logSubstitutions[key]
524                     value = value.replace("%log%", message)
525                     if value != "@remove@":
526                         result += line.replace(key, value) + "\n"
527                     substituted = True
528                     break
529
530             if not substituted:
531                 result += line + "\n"
532
533         return result
534
535     def prepareSubmitTemplate(self):
536         # remove lines in the Files section that show changes to files outside the depot path we're committing into
537         template = ""
538         inFilesSection = False
539         for line in read_pipe_lines("p4 change -o"):
540             if inFilesSection:
541                 if line.startswith("\t"):
542                     # path starts and ends with a tab
543                     path = line[1:]
544                     lastTab = path.rfind("\t")
545                     if lastTab != -1:
546                         path = path[:lastTab]
547                         if not path.startswith(self.depotPath):
548                             continue
549                 else:
550                     inFilesSection = False
551             else:
552                 if line.startswith("Files:"):
553                     inFilesSection = True
554
555             template += line
556
557         return template
558
559     def applyCommit(self, id):
560         if self.directSubmit:
561             print "Applying local change in working directory/index"
562             diff = self.diffStatus
563         else:
564             print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
565             diffOpts = ("", "-M")[self.detectRename]
566             diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
567         filesToAdd = set()
568         filesToDelete = set()
569         editedFiles = set()
570         filesToChangeExecBit = {}
571         for line in diff:
572             diff = parseDiffTreeEntry(line)
573             modifier = diff['status']
574             path = diff['src']
575             if modifier == "M":
576                 system("p4 edit \"%s\"" % path)
577                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
578                     filesToChangeExecBit[path] = diff['dst_mode']
579                 editedFiles.add(path)
580             elif modifier == "A":
581                 filesToAdd.add(path)
582                 filesToChangeExecBit[path] = diff['dst_mode']
583                 if path in filesToDelete:
584                     filesToDelete.remove(path)
585             elif modifier == "D":
586                 filesToDelete.add(path)
587                 if path in filesToAdd:
588                     filesToAdd.remove(path)
589             elif modifier == "R":
590                 src, dest = diff['src'], diff['dst']
591                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
592                 system("p4 edit \"%s\"" % (dest))
593                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
594                     filesToChangeExecBit[dest] = diff['dst_mode']
595                 os.unlink(dest)
596                 editedFiles.add(dest)
597                 filesToDelete.add(src)
598             else:
599                 die("unknown modifier %s for %s" % (modifier, path))
600
601         if self.directSubmit:
602             diffcmd = "cat \"%s\"" % self.diffFile
603         else:
604             diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
605         patchcmd = diffcmd + " | git apply "
606         tryPatchCmd = patchcmd + "--check -"
607         applyPatchCmd = patchcmd + "--check --apply -"
608
609         if os.system(tryPatchCmd) != 0:
610             print "Unfortunately applying the change failed!"
611             print "What do you want to do?"
612             response = "x"
613             while response != "s" and response != "a" and response != "w":
614                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
615                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
616             if response == "s":
617                 print "Skipping! Good luck with the next patches..."
618                 for f in editedFiles:
619                     system("p4 revert \"%s\"" % f);
620                 for f in filesToAdd:
621                     system("rm %s" %f)
622                 return
623             elif response == "a":
624                 os.system(applyPatchCmd)
625                 if len(filesToAdd) > 0:
626                     print "You may also want to call p4 add on the following files:"
627                     print " ".join(filesToAdd)
628                 if len(filesToDelete):
629                     print "The following files should be scheduled for deletion with p4 delete:"
630                     print " ".join(filesToDelete)
631                 die("Please resolve and submit the conflict manually and "
632                     + "continue afterwards with git-p4 submit --continue")
633             elif response == "w":
634                 system(diffcmd + " > patch.txt")
635                 print "Patch saved to patch.txt in %s !" % self.clientPath
636                 die("Please resolve and submit the conflict manually and "
637                     "continue afterwards with git-p4 submit --continue")
638
639         system(applyPatchCmd)
640
641         for f in filesToAdd:
642             system("p4 add \"%s\"" % f)
643         for f in filesToDelete:
644             system("p4 revert \"%s\"" % f)
645             system("p4 delete \"%s\"" % f)
646
647         # Set/clear executable bits
648         for f in filesToChangeExecBit.keys():
649             mode = filesToChangeExecBit[f]
650             setP4ExecBit(f, mode)
651
652         logMessage = ""
653         if not self.directSubmit:
654             logMessage = extractLogMessageFromGitCommit(id)
655             logMessage = logMessage.replace("\n", "\n\t")
656             if self.isWindows:
657                 logMessage = logMessage.replace("\n", "\r\n")
658             logMessage = logMessage.strip()
659
660         template = self.prepareSubmitTemplate()
661
662         if self.interactive:
663             submitTemplate = self.prepareLogMessage(template, logMessage)
664             diff = read_pipe("p4 diff -du ...")
665
666             for newFile in filesToAdd:
667                 diff += "==== new file ====\n"
668                 diff += "--- /dev/null\n"
669                 diff += "+++ %s\n" % newFile
670                 f = open(newFile, "r")
671                 for line in f.readlines():
672                     diff += "+" + line
673                 f.close()
674
675             separatorLine = "######## everything below this line is just the diff #######"
676             if platform.system() == "Windows":
677                 separatorLine += "\r"
678             separatorLine += "\n"
679
680             [handle, fileName] = tempfile.mkstemp()
681             tmpFile = os.fdopen(handle, "w+")
682             tmpFile.write(submitTemplate + separatorLine + diff)
683             tmpFile.close()
684             defaultEditor = "vi"
685             if platform.system() == "Windows":
686                 defaultEditor = "notepad"
687             editor = os.environ.get("EDITOR", defaultEditor);
688             system(editor + " " + fileName)
689             tmpFile = open(fileName, "rb")
690             message = tmpFile.read()
691             tmpFile.close()
692             os.remove(fileName)
693             submitTemplate = message[:message.index(separatorLine)]
694             if self.isWindows:
695                 submitTemplate = submitTemplate.replace("\r\n", "\n")
696
697             if self.directSubmit:
698                 print "Submitting to git first"
699                 os.chdir(self.oldWorkingDirectory)
700                 write_pipe("git commit -a -F -", submitTemplate)
701                 os.chdir(self.clientPath)
702
703             write_pipe("p4 submit -i", submitTemplate)
704         else:
705             fileName = "submit.txt"
706             file = open(fileName, "w+")
707             file.write(self.prepareLogMessage(template, logMessage))
708             file.close()
709             print ("Perforce submit template written as %s. "
710                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
711                    % (fileName, fileName))
712
713     def run(self, args):
714         if len(args) == 0:
715             self.master = currentGitBranch()
716             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
717                 die("Detecting current git branch failed!")
718         elif len(args) == 1:
719             self.master = args[0]
720         else:
721             return False
722
723         [upstream, settings] = findUpstreamBranchPoint()
724         self.depotPath = settings['depot-paths'][0]
725         if len(self.origin) == 0:
726             self.origin = upstream
727
728         if self.verbose:
729             print "Origin branch is " + self.origin
730
731         if len(self.depotPath) == 0:
732             print "Internal error: cannot locate perforce depot path from existing branches"
733             sys.exit(128)
734
735         self.clientPath = p4Where(self.depotPath)
736
737         if len(self.clientPath) == 0:
738             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
739             sys.exit(128)
740
741         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
742         self.oldWorkingDirectory = os.getcwd()
743
744         if self.directSubmit:
745             self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
746             if len(self.diffStatus) == 0:
747                 print "No changes in working directory to submit."
748                 return True
749             patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
750             self.diffFile = self.gitdir + "/p4-git-diff"
751             f = open(self.diffFile, "wb")
752             f.write(patch)
753             f.close();
754
755         os.chdir(self.clientPath)
756         print "Syncronizing p4 checkout..."
757         system("p4 sync ...")
758
759         if self.reset:
760             self.firstTime = True
761
762         if len(self.substFile) > 0:
763             for line in open(self.substFile, "r").readlines():
764                 tokens = line.strip().split("=")
765                 self.logSubstitutions[tokens[0]] = tokens[1]
766
767         self.check()
768         self.configFile = self.gitdir + "/p4-git-sync.cfg"
769         self.config = shelve.open(self.configFile, writeback=True)
770
771         if self.firstTime:
772             self.start()
773
774         commits = self.config.get("commits", [])
775
776         while len(commits) > 0:
777             self.firstTime = False
778             commit = commits[0]
779             commits = commits[1:]
780             self.config["commits"] = commits
781             self.applyCommit(commit)
782             if not self.interactive:
783                 break
784
785         self.config.close()
786
787         if self.directSubmit:
788             os.remove(self.diffFile)
789
790         if len(commits) == 0:
791             if self.firstTime:
792                 print "No changes found to apply between %s and current HEAD" % self.origin
793             else:
794                 print "All changes applied!"
795                 os.chdir(self.oldWorkingDirectory)
796
797                 sync = P4Sync()
798                 sync.run([])
799
800                 rebase = P4Rebase()
801                 rebase.rebase()
802             os.remove(self.configFile)
803
804         return True
805
806 class P4Sync(Command):
807     def __init__(self):
808         Command.__init__(self)
809         self.options = [
810                 optparse.make_option("--branch", dest="branch"),
811                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
812                 optparse.make_option("--changesfile", dest="changesFile"),
813                 optparse.make_option("--silent", dest="silent", action="store_true"),
814                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
815                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
816                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
817                                      help="Import into refs/heads/ , not refs/remotes"),
818                 optparse.make_option("--max-changes", dest="maxChanges"),
819                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
820                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
821         ]
822         self.description = """Imports from Perforce into a git repository.\n
823     example:
824     //depot/my/project/ -- to import the current head
825     //depot/my/project/@all -- to import everything
826     //depot/my/project/@1,6 -- to import only from revision 1 to 6
827
828     (a ... is not needed in the path p4 specification, it's added implicitly)"""
829
830         self.usage += " //depot/path[@revRange]"
831         self.silent = False
832         self.createdBranches = Set()
833         self.committedChanges = Set()
834         self.branch = ""
835         self.detectBranches = False
836         self.detectLabels = False
837         self.changesFile = ""
838         self.syncWithOrigin = True
839         self.verbose = False
840         self.importIntoRemotes = True
841         self.maxChanges = ""
842         self.isWindows = (platform.system() == "Windows")
843         self.keepRepoPath = False
844         self.depotPaths = None
845         self.p4BranchesInGit = []
846
847         if gitConfig("git-p4.syncFromOrigin") == "false":
848             self.syncWithOrigin = False
849
850     def extractFilesFromCommit(self, commit):
851         files = []
852         fnum = 0
853         while commit.has_key("depotFile%s" % fnum):
854             path =  commit["depotFile%s" % fnum]
855
856             found = [p for p in self.depotPaths
857                      if path.startswith (p)]
858             if not found:
859                 fnum = fnum + 1
860                 continue
861
862             file = {}
863             file["path"] = path
864             file["rev"] = commit["rev%s" % fnum]
865             file["action"] = commit["action%s" % fnum]
866             file["type"] = commit["type%s" % fnum]
867             files.append(file)
868             fnum = fnum + 1
869         return files
870
871     def stripRepoPath(self, path, prefixes):
872         if self.keepRepoPath:
873             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
874
875         for p in prefixes:
876             if path.startswith(p):
877                 path = path[len(p):]
878
879         return path
880
881     def splitFilesIntoBranches(self, commit):
882         branches = {}
883         fnum = 0
884         while commit.has_key("depotFile%s" % fnum):
885             path =  commit["depotFile%s" % fnum]
886             found = [p for p in self.depotPaths
887                      if path.startswith (p)]
888             if not found:
889                 fnum = fnum + 1
890                 continue
891
892             file = {}
893             file["path"] = path
894             file["rev"] = commit["rev%s" % fnum]
895             file["action"] = commit["action%s" % fnum]
896             file["type"] = commit["type%s" % fnum]
897             fnum = fnum + 1
898
899             relPath = self.stripRepoPath(path, self.depotPaths)
900
901             for branch in self.knownBranches.keys():
902
903                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
904                 if relPath.startswith(branch + "/"):
905                     if branch not in branches:
906                         branches[branch] = []
907                     branches[branch].append(file)
908                     break
909
910         return branches
911
912     ## Should move this out, doesn't use SELF.
913     def readP4Files(self, files):
914         files = [f for f in files
915                  if f['action'] != 'delete']
916
917         if not files:
918             return
919
920         filedata = p4CmdList('-x - print',
921                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
922                                               for f in files]),
923                              stdin_mode='w+')
924         if "p4ExitCode" in filedata[0]:
925             die("Problems executing p4. Error: [%d]."
926                 % (filedata[0]['p4ExitCode']));
927
928         j = 0;
929         contents = {}
930         while j < len(filedata):
931             stat = filedata[j]
932             j += 1
933             text = ''
934             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
935                 tmp = filedata[j]['data']
936                 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
937                     tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
938                 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
939                     tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
940                 text += tmp
941                 j += 1
942
943
944             if not stat.has_key('depotFile'):
945                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
946                 continue
947
948             contents[stat['depotFile']] = text
949
950         for f in files:
951             assert not f.has_key('data')
952             f['data'] = contents[f['path']]
953
954     def commit(self, details, files, branch, branchPrefixes, parent = ""):
955         epoch = details["time"]
956         author = details["user"]
957
958         if self.verbose:
959             print "commit into %s" % branch
960
961         # start with reading files; if that fails, we should not
962         # create a commit.
963         new_files = []
964         for f in files:
965             if [p for p in branchPrefixes if f['path'].startswith(p)]:
966                 new_files.append (f)
967             else:
968                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
969         files = new_files
970         self.readP4Files(files)
971
972
973
974
975         self.gitStream.write("commit %s\n" % branch)
976 #        gitStream.write("mark :%s\n" % details["change"])
977         self.committedChanges.add(int(details["change"]))
978         committer = ""
979         if author not in self.users:
980             self.getUserMapFromPerforceServer()
981         if author in self.users:
982             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
983         else:
984             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
985
986         self.gitStream.write("committer %s\n" % committer)
987
988         self.gitStream.write("data <<EOT\n")
989         self.gitStream.write(details["desc"])
990         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
991                              % (','.join (branchPrefixes), details["change"]))
992         if len(details['options']) > 0:
993             self.gitStream.write(": options = %s" % details['options'])
994         self.gitStream.write("]\nEOT\n\n")
995
996         if len(parent) > 0:
997             if self.verbose:
998                 print "parent %s" % parent
999             self.gitStream.write("from %s\n" % parent)
1000
1001         for file in files:
1002             if file["type"] == "apple":
1003                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1004                 continue
1005
1006             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1007             if file["action"] == "delete":
1008                 self.gitStream.write("D %s\n" % relPath)
1009             else:
1010                 data = file['data']
1011
1012                 mode = "644"
1013                 if isP4Exec(file["type"]):
1014                     mode = "755"
1015                 elif file["type"] == "symlink":
1016                     mode = "120000"
1017                     # p4 print on a symlink contains "target\n", so strip it off
1018                     data = data[:-1]
1019
1020                 if self.isWindows and file["type"].endswith("text"):
1021                     data = data.replace("\r\n", "\n")
1022
1023                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1024                 self.gitStream.write("data %s\n" % len(data))
1025                 self.gitStream.write(data)
1026                 self.gitStream.write("\n")
1027
1028         self.gitStream.write("\n")
1029
1030         change = int(details["change"])
1031
1032         if self.labels.has_key(change):
1033             label = self.labels[change]
1034             labelDetails = label[0]
1035             labelRevisions = label[1]
1036             if self.verbose:
1037                 print "Change %s is labelled %s" % (change, labelDetails)
1038
1039             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1040                                                     for p in branchPrefixes]))
1041
1042             if len(files) == len(labelRevisions):
1043
1044                 cleanedFiles = {}
1045                 for info in files:
1046                     if info["action"] == "delete":
1047                         continue
1048                     cleanedFiles[info["depotFile"]] = info["rev"]
1049
1050                 if cleanedFiles == labelRevisions:
1051                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1052                     self.gitStream.write("from %s\n" % branch)
1053
1054                     owner = labelDetails["Owner"]
1055                     tagger = ""
1056                     if author in self.users:
1057                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1058                     else:
1059                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1060                     self.gitStream.write("tagger %s\n" % tagger)
1061                     self.gitStream.write("data <<EOT\n")
1062                     self.gitStream.write(labelDetails["Description"])
1063                     self.gitStream.write("EOT\n\n")
1064
1065                 else:
1066                     if not self.silent:
1067                         print ("Tag %s does not match with change %s: files do not match."
1068                                % (labelDetails["label"], change))
1069
1070             else:
1071                 if not self.silent:
1072                     print ("Tag %s does not match with change %s: file count is different."
1073                            % (labelDetails["label"], change))
1074
1075     def getUserCacheFilename(self):
1076         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1077         return home + "/.gitp4-usercache.txt"
1078
1079     def getUserMapFromPerforceServer(self):
1080         if self.userMapFromPerforceServer:
1081             return
1082         self.users = {}
1083
1084         for output in p4CmdList("users"):
1085             if not output.has_key("User"):
1086                 continue
1087             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1088
1089
1090         s = ''
1091         for (key, val) in self.users.items():
1092             s += "%s\t%s\n" % (key, val)
1093
1094         open(self.getUserCacheFilename(), "wb").write(s)
1095         self.userMapFromPerforceServer = True
1096
1097     def loadUserMapFromCache(self):
1098         self.users = {}
1099         self.userMapFromPerforceServer = False
1100         try:
1101             cache = open(self.getUserCacheFilename(), "rb")
1102             lines = cache.readlines()
1103             cache.close()
1104             for line in lines:
1105                 entry = line.strip().split("\t")
1106                 self.users[entry[0]] = entry[1]
1107         except IOError:
1108             self.getUserMapFromPerforceServer()
1109
1110     def getLabels(self):
1111         self.labels = {}
1112
1113         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1114         if len(l) > 0 and not self.silent:
1115             print "Finding files belonging to labels in %s" % `self.depotPaths`
1116
1117         for output in l:
1118             label = output["label"]
1119             revisions = {}
1120             newestChange = 0
1121             if self.verbose:
1122                 print "Querying files for label %s" % label
1123             for file in p4CmdList("files "
1124                                   +  ' '.join (["%s...@%s" % (p, label)
1125                                                 for p in self.depotPaths])):
1126                 revisions[file["depotFile"]] = file["rev"]
1127                 change = int(file["change"])
1128                 if change > newestChange:
1129                     newestChange = change
1130
1131             self.labels[newestChange] = [output, revisions]
1132
1133         if self.verbose:
1134             print "Label changes: %s" % self.labels.keys()
1135
1136     def guessProjectName(self):
1137         for p in self.depotPaths:
1138             if p.endswith("/"):
1139                 p = p[:-1]
1140             p = p[p.strip().rfind("/") + 1:]
1141             if not p.endswith("/"):
1142                p += "/"
1143             return p
1144
1145     def getBranchMapping(self):
1146         lostAndFoundBranches = set()
1147
1148         for info in p4CmdList("branches"):
1149             details = p4Cmd("branch -o %s" % info["branch"])
1150             viewIdx = 0
1151             while details.has_key("View%s" % viewIdx):
1152                 paths = details["View%s" % viewIdx].split(" ")
1153                 viewIdx = viewIdx + 1
1154                 # require standard //depot/foo/... //depot/bar/... mapping
1155                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1156                     continue
1157                 source = paths[0]
1158                 destination = paths[1]
1159                 ## HACK
1160                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1161                     source = source[len(self.depotPaths[0]):-4]
1162                     destination = destination[len(self.depotPaths[0]):-4]
1163
1164                     if destination in self.knownBranches:
1165                         if not self.silent:
1166                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1167                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1168                         continue
1169
1170                     self.knownBranches[destination] = source
1171
1172                     lostAndFoundBranches.discard(destination)
1173
1174                     if source not in self.knownBranches:
1175                         lostAndFoundBranches.add(source)
1176
1177
1178         for branch in lostAndFoundBranches:
1179             self.knownBranches[branch] = branch
1180
1181     def getBranchMappingFromGitBranches(self):
1182         branches = p4BranchesInGit(self.importIntoRemotes)
1183         for branch in branches.keys():
1184             if branch == "master":
1185                 branch = "main"
1186             else:
1187                 branch = branch[len(self.projectName):]
1188             self.knownBranches[branch] = branch
1189
1190     def listExistingP4GitBranches(self):
1191         # branches holds mapping from name to commit
1192         branches = p4BranchesInGit(self.importIntoRemotes)
1193         self.p4BranchesInGit = branches.keys()
1194         for branch in branches.keys():
1195             self.initialParents[self.refPrefix + branch] = branches[branch]
1196
1197     def updateOptionDict(self, d):
1198         option_keys = {}
1199         if self.keepRepoPath:
1200             option_keys['keepRepoPath'] = 1
1201
1202         d["options"] = ' '.join(sorted(option_keys.keys()))
1203
1204     def readOptions(self, d):
1205         self.keepRepoPath = (d.has_key('options')
1206                              and ('keepRepoPath' in d['options']))
1207
1208     def gitRefForBranch(self, branch):
1209         if branch == "main":
1210             return self.refPrefix + "master"
1211
1212         if len(branch) <= 0:
1213             return branch
1214
1215         return self.refPrefix + self.projectName + branch
1216
1217     def gitCommitByP4Change(self, ref, change):
1218         if self.verbose:
1219             print "looking in ref " + ref + " for change %s using bisect..." % change
1220
1221         earliestCommit = ""
1222         latestCommit = parseRevision(ref)
1223
1224         while True:
1225             if self.verbose:
1226                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1227             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1228             if len(next) == 0:
1229                 if self.verbose:
1230                     print "argh"
1231                 return ""
1232             log = extractLogMessageFromGitCommit(next)
1233             settings = extractSettingsGitLog(log)
1234             currentChange = int(settings['change'])
1235             if self.verbose:
1236                 print "current change %s" % currentChange
1237
1238             if currentChange == change:
1239                 if self.verbose:
1240                     print "found %s" % next
1241                 return next
1242
1243             if currentChange < change:
1244                 earliestCommit = "^%s" % next
1245             else:
1246                 latestCommit = "%s" % next
1247
1248         return ""
1249
1250     def importNewBranch(self, branch, maxChange):
1251         # make fast-import flush all changes to disk and update the refs using the checkpoint
1252         # command so that we can try to find the branch parent in the git history
1253         self.gitStream.write("checkpoint\n\n");
1254         self.gitStream.flush();
1255         branchPrefix = self.depotPaths[0] + branch + "/"
1256         range = "@1,%s" % maxChange
1257         #print "prefix" + branchPrefix
1258         changes = p4ChangesForPaths([branchPrefix], range)
1259         if len(changes) <= 0:
1260             return False
1261         firstChange = changes[0]
1262         #print "first change in branch: %s" % firstChange
1263         sourceBranch = self.knownBranches[branch]
1264         sourceDepotPath = self.depotPaths[0] + sourceBranch
1265         sourceRef = self.gitRefForBranch(sourceBranch)
1266         #print "source " + sourceBranch
1267
1268         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1269         #print "branch parent: %s" % branchParentChange
1270         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1271         if len(gitParent) > 0:
1272             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1273             #print "parent git commit: %s" % gitParent
1274
1275         self.importChanges(changes)
1276         return True
1277
1278     def importChanges(self, changes):
1279         cnt = 1
1280         for change in changes:
1281             description = p4Cmd("describe %s" % change)
1282             self.updateOptionDict(description)
1283
1284             if not self.silent:
1285                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1286                 sys.stdout.flush()
1287             cnt = cnt + 1
1288
1289             try:
1290                 if self.detectBranches:
1291                     branches = self.splitFilesIntoBranches(description)
1292                     for branch in branches.keys():
1293                         ## HACK  --hwn
1294                         branchPrefix = self.depotPaths[0] + branch + "/"
1295
1296                         parent = ""
1297
1298                         filesForCommit = branches[branch]
1299
1300                         if self.verbose:
1301                             print "branch is %s" % branch
1302
1303                         self.updatedBranches.add(branch)
1304
1305                         if branch not in self.createdBranches:
1306                             self.createdBranches.add(branch)
1307                             parent = self.knownBranches[branch]
1308                             if parent == branch:
1309                                 parent = ""
1310                             else:
1311                                 fullBranch = self.projectName + branch
1312                                 if fullBranch not in self.p4BranchesInGit:
1313                                     if not self.silent:
1314                                         print("\n    Importing new branch %s" % fullBranch);
1315                                     if self.importNewBranch(branch, change - 1):
1316                                         parent = ""
1317                                         self.p4BranchesInGit.append(fullBranch)
1318                                     if not self.silent:
1319                                         print("\n    Resuming with change %s" % change);
1320
1321                                 if self.verbose:
1322                                     print "parent determined through known branches: %s" % parent
1323
1324                         branch = self.gitRefForBranch(branch)
1325                         parent = self.gitRefForBranch(parent)
1326
1327                         if self.verbose:
1328                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1329
1330                         if len(parent) == 0 and branch in self.initialParents:
1331                             parent = self.initialParents[branch]
1332                             del self.initialParents[branch]
1333
1334                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1335                 else:
1336                     files = self.extractFilesFromCommit(description)
1337                     self.commit(description, files, self.branch, self.depotPaths,
1338                                 self.initialParent)
1339                     self.initialParent = ""
1340             except IOError:
1341                 print self.gitError.read()
1342                 sys.exit(1)
1343
1344     def importHeadRevision(self, revision):
1345         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1346
1347         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1348         details["desc"] = ("Initial import of %s from the state at revision %s"
1349                            % (' '.join(self.depotPaths), revision))
1350         details["change"] = revision
1351         newestRevision = 0
1352
1353         fileCnt = 0
1354         for info in p4CmdList("files "
1355                               +  ' '.join(["%s...%s"
1356                                            % (p, revision)
1357                                            for p in self.depotPaths])):
1358
1359             if info['code'] == 'error':
1360                 sys.stderr.write("p4 returned an error: %s\n"
1361                                  % info['data'])
1362                 sys.exit(1)
1363
1364
1365             change = int(info["change"])
1366             if change > newestRevision:
1367                 newestRevision = change
1368
1369             if info["action"] == "delete":
1370                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1371                 #fileCnt = fileCnt + 1
1372                 continue
1373
1374             for prop in ["depotFile", "rev", "action", "type" ]:
1375                 details["%s%s" % (prop, fileCnt)] = info[prop]
1376
1377             fileCnt = fileCnt + 1
1378
1379         details["change"] = newestRevision
1380         self.updateOptionDict(details)
1381         try:
1382             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1383         except IOError:
1384             print "IO error with git fast-import. Is your git version recent enough?"
1385             print self.gitError.read()
1386
1387
1388     def run(self, args):
1389         self.depotPaths = []
1390         self.changeRange = ""
1391         self.initialParent = ""
1392         self.previousDepotPaths = []
1393
1394         # map from branch depot path to parent branch
1395         self.knownBranches = {}
1396         self.initialParents = {}
1397         self.hasOrigin = originP4BranchesExist()
1398         if not self.syncWithOrigin:
1399             self.hasOrigin = False
1400
1401         if self.importIntoRemotes:
1402             self.refPrefix = "refs/remotes/p4/"
1403         else:
1404             self.refPrefix = "refs/heads/p4/"
1405
1406         if self.syncWithOrigin and self.hasOrigin:
1407             if not self.silent:
1408                 print "Syncing with origin first by calling git fetch origin"
1409             system("git fetch origin")
1410
1411         if len(self.branch) == 0:
1412             self.branch = self.refPrefix + "master"
1413             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1414                 system("git update-ref %s refs/heads/p4" % self.branch)
1415                 system("git branch -D p4");
1416             # create it /after/ importing, when master exists
1417             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1418                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1419
1420         # TODO: should always look at previous commits,
1421         # merge with previous imports, if possible.
1422         if args == []:
1423             if self.hasOrigin:
1424                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1425             self.listExistingP4GitBranches()
1426
1427             if len(self.p4BranchesInGit) > 1:
1428                 if not self.silent:
1429                     print "Importing from/into multiple branches"
1430                 self.detectBranches = True
1431
1432             if self.verbose:
1433                 print "branches: %s" % self.p4BranchesInGit
1434
1435             p4Change = 0
1436             for branch in self.p4BranchesInGit:
1437                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1438
1439                 settings = extractSettingsGitLog(logMsg)
1440
1441                 self.readOptions(settings)
1442                 if (settings.has_key('depot-paths')
1443                     and settings.has_key ('change')):
1444                     change = int(settings['change']) + 1
1445                     p4Change = max(p4Change, change)
1446
1447                     depotPaths = sorted(settings['depot-paths'])
1448                     if self.previousDepotPaths == []:
1449                         self.previousDepotPaths = depotPaths
1450                     else:
1451                         paths = []
1452                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1453                             for i in range(0, min(len(cur), len(prev))):
1454                                 if cur[i] <> prev[i]:
1455                                     i = i - 1
1456                                     break
1457
1458                             paths.append (cur[:i + 1])
1459
1460                         self.previousDepotPaths = paths
1461
1462             if p4Change > 0:
1463                 self.depotPaths = sorted(self.previousDepotPaths)
1464                 self.changeRange = "@%s,#head" % p4Change
1465                 if not self.detectBranches:
1466                     self.initialParent = parseRevision(self.branch)
1467                 if not self.silent and not self.detectBranches:
1468                     print "Performing incremental import into %s git branch" % self.branch
1469
1470         if not self.branch.startswith("refs/"):
1471             self.branch = "refs/heads/" + self.branch
1472
1473         if len(args) == 0 and self.depotPaths:
1474             if not self.silent:
1475                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1476         else:
1477             if self.depotPaths and self.depotPaths != args:
1478                 print ("previous import used depot path %s and now %s was specified. "
1479                        "This doesn't work!" % (' '.join (self.depotPaths),
1480                                                ' '.join (args)))
1481                 sys.exit(1)
1482
1483             self.depotPaths = sorted(args)
1484
1485         revision = ""
1486         self.users = {}
1487
1488         newPaths = []
1489         for p in self.depotPaths:
1490             if p.find("@") != -1:
1491                 atIdx = p.index("@")
1492                 self.changeRange = p[atIdx:]
1493                 if self.changeRange == "@all":
1494                     self.changeRange = ""
1495                 elif ',' not in self.changeRange:
1496                     revision = self.changeRange
1497                     self.changeRange = ""
1498                 p = p[:atIdx]
1499             elif p.find("#") != -1:
1500                 hashIdx = p.index("#")
1501                 revision = p[hashIdx:]
1502                 p = p[:hashIdx]
1503             elif self.previousDepotPaths == []:
1504                 revision = "#head"
1505
1506             p = re.sub ("\.\.\.$", "", p)
1507             if not p.endswith("/"):
1508                 p += "/"
1509
1510             newPaths.append(p)
1511
1512         self.depotPaths = newPaths
1513
1514
1515         self.loadUserMapFromCache()
1516         self.labels = {}
1517         if self.detectLabels:
1518             self.getLabels();
1519
1520         if self.detectBranches:
1521             ## FIXME - what's a P4 projectName ?
1522             self.projectName = self.guessProjectName()
1523
1524             if self.hasOrigin:
1525                 self.getBranchMappingFromGitBranches()
1526             else:
1527                 self.getBranchMapping()
1528             if self.verbose:
1529                 print "p4-git branches: %s" % self.p4BranchesInGit
1530                 print "initial parents: %s" % self.initialParents
1531             for b in self.p4BranchesInGit:
1532                 if b != "master":
1533
1534                     ## FIXME
1535                     b = b[len(self.projectName):]
1536                 self.createdBranches.add(b)
1537
1538         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1539
1540         importProcess = subprocess.Popen(["git", "fast-import"],
1541                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1542                                          stderr=subprocess.PIPE);
1543         self.gitOutput = importProcess.stdout
1544         self.gitStream = importProcess.stdin
1545         self.gitError = importProcess.stderr
1546
1547         if revision:
1548             self.importHeadRevision(revision)
1549         else:
1550             changes = []
1551
1552             if len(self.changesFile) > 0:
1553                 output = open(self.changesFile).readlines()
1554                 changeSet = Set()
1555                 for line in output:
1556                     changeSet.add(int(line))
1557
1558                 for change in changeSet:
1559                     changes.append(change)
1560
1561                 changes.sort()
1562             else:
1563                 if self.verbose:
1564                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1565                                                               self.changeRange)
1566                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1567
1568                 if len(self.maxChanges) > 0:
1569                     changes = changes[:min(int(self.maxChanges), len(changes))]
1570
1571             if len(changes) == 0:
1572                 if not self.silent:
1573                     print "No changes to import!"
1574                 return True
1575
1576             if not self.silent and not self.detectBranches:
1577                 print "Import destination: %s" % self.branch
1578
1579             self.updatedBranches = set()
1580
1581             self.importChanges(changes)
1582
1583             if not self.silent:
1584                 print ""
1585                 if len(self.updatedBranches) > 0:
1586                     sys.stdout.write("Updated branches: ")
1587                     for b in self.updatedBranches:
1588                         sys.stdout.write("%s " % b)
1589                     sys.stdout.write("\n")
1590
1591         self.gitStream.close()
1592         if importProcess.wait() != 0:
1593             die("fast-import failed: %s" % self.gitError.read())
1594         self.gitOutput.close()
1595         self.gitError.close()
1596
1597         return True
1598
1599 class P4Rebase(Command):
1600     def __init__(self):
1601         Command.__init__(self)
1602         self.options = [ ]
1603         self.description = ("Fetches the latest revision from perforce and "
1604                             + "rebases the current work (branch) against it")
1605         self.verbose = False
1606
1607     def run(self, args):
1608         sync = P4Sync()
1609         sync.run([])
1610
1611         return self.rebase()
1612
1613     def rebase(self):
1614         if os.system("git update-index --refresh") != 0:
1615             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1616         if len(read_pipe("git diff-index HEAD --")) > 0:
1617             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1618
1619         [upstream, settings] = findUpstreamBranchPoint()
1620         if len(upstream) == 0:
1621             die("Cannot find upstream branchpoint for rebase")
1622
1623         # the branchpoint may be p4/foo~3, so strip off the parent
1624         upstream = re.sub("~[0-9]+$", "", upstream)
1625
1626         print "Rebasing the current branch onto %s" % upstream
1627         oldHead = read_pipe("git rev-parse HEAD").strip()
1628         system("git rebase %s" % upstream)
1629         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1630         return True
1631
1632 class P4Clone(P4Sync):
1633     def __init__(self):
1634         P4Sync.__init__(self)
1635         self.description = "Creates a new git repository and imports from Perforce into it"
1636         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1637         self.options.append(
1638             optparse.make_option("--destination", dest="cloneDestination",
1639                                  action='store', default=None,
1640                                  help="where to leave result of the clone"))
1641         self.cloneDestination = None
1642         self.needsGit = False
1643
1644     def defaultDestination(self, args):
1645         ## TODO: use common prefix of args?
1646         depotPath = args[0]
1647         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1648         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1649         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1650         depotDir = re.sub(r"/$", "", depotDir)
1651         return os.path.split(depotDir)[1]
1652
1653     def run(self, args):
1654         if len(args) < 1:
1655             return False
1656
1657         if self.keepRepoPath and not self.cloneDestination:
1658             sys.stderr.write("Must specify destination for --keep-path\n")
1659             sys.exit(1)
1660
1661         depotPaths = args
1662
1663         if not self.cloneDestination and len(depotPaths) > 1:
1664             self.cloneDestination = depotPaths[-1]
1665             depotPaths = depotPaths[:-1]
1666
1667         for p in depotPaths:
1668             if not p.startswith("//"):
1669                 return False
1670
1671         if not self.cloneDestination:
1672             self.cloneDestination = self.defaultDestination(args)
1673
1674         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1675         if not os.path.exists(self.cloneDestination):
1676             os.makedirs(self.cloneDestination)
1677         os.chdir(self.cloneDestination)
1678         system("git init")
1679         self.gitdir = os.getcwd() + "/.git"
1680         if not P4Sync.run(self, depotPaths):
1681             return False
1682         if self.branch != "master":
1683             if gitBranchExists("refs/remotes/p4/master"):
1684                 system("git branch master refs/remotes/p4/master")
1685                 system("git checkout -f")
1686             else:
1687                 print "Could not detect main branch. No checkout/master branch created."
1688
1689         return True
1690
1691 class P4Branches(Command):
1692     def __init__(self):
1693         Command.__init__(self)
1694         self.options = [ ]
1695         self.description = ("Shows the git branches that hold imports and their "
1696                             + "corresponding perforce depot paths")
1697         self.verbose = False
1698
1699     def run(self, args):
1700         if originP4BranchesExist():
1701             createOrUpdateBranchesFromOrigin()
1702
1703         cmdline = "git rev-parse --symbolic "
1704         cmdline += " --remotes"
1705
1706         for line in read_pipe_lines(cmdline):
1707             line = line.strip()
1708
1709             if not line.startswith('p4/') or line == "p4/HEAD":
1710                 continue
1711             branch = line
1712
1713             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1714             settings = extractSettingsGitLog(log)
1715
1716             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1717         return True
1718
1719 class HelpFormatter(optparse.IndentedHelpFormatter):
1720     def __init__(self):
1721         optparse.IndentedHelpFormatter.__init__(self)
1722
1723     def format_description(self, description):
1724         if description:
1725             return description + "\n"
1726         else:
1727             return ""
1728
1729 def printUsage(commands):
1730     print "usage: %s <command> [options]" % sys.argv[0]
1731     print ""
1732     print "valid commands: %s" % ", ".join(commands)
1733     print ""
1734     print "Try %s <command> --help for command specific help." % sys.argv[0]
1735     print ""
1736
1737 commands = {
1738     "debug" : P4Debug,
1739     "submit" : P4Submit,
1740     "commit" : P4Submit,
1741     "sync" : P4Sync,
1742     "rebase" : P4Rebase,
1743     "clone" : P4Clone,
1744     "rollback" : P4RollBack,
1745     "branches" : P4Branches
1746 }
1747
1748
1749 def main():
1750     if len(sys.argv[1:]) == 0:
1751         printUsage(commands.keys())
1752         sys.exit(2)
1753
1754     cmd = ""
1755     cmdName = sys.argv[1]
1756     try:
1757         klass = commands[cmdName]
1758         cmd = klass()
1759     except KeyError:
1760         print "unknown command %s" % cmdName
1761         print ""
1762         printUsage(commands.keys())
1763         sys.exit(2)
1764
1765     options = cmd.options
1766     cmd.gitdir = os.environ.get("GIT_DIR", None)
1767
1768     args = sys.argv[2:]
1769
1770     if len(options) > 0:
1771         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1772
1773         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1774                                        options,
1775                                        description = cmd.description,
1776                                        formatter = HelpFormatter())
1777
1778         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1779     global verbose
1780     verbose = cmd.verbose
1781     if cmd.needsGit:
1782         if cmd.gitdir == None:
1783             cmd.gitdir = os.path.abspath(".git")
1784             if not isValidGitDir(cmd.gitdir):
1785                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1786                 if os.path.exists(cmd.gitdir):
1787                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1788                     if len(cdup) > 0:
1789                         os.chdir(cdup);
1790
1791         if not isValidGitDir(cmd.gitdir):
1792             if isValidGitDir(cmd.gitdir + "/.git"):
1793                 cmd.gitdir += "/.git"
1794             else:
1795                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1796
1797         os.environ["GIT_DIR"] = cmd.gitdir
1798
1799     if not cmd.run(args):
1800         parser.print_help()
1801
1802
1803 if __name__ == '__main__':
1804     main()