git.oblomov.eu Git - git/blob - contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRename = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616         diffOpts = ("", "-M")[self.detectRename]
 617         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618         filesToAdd = set()
 619         filesToDelete = set()
 620         editedFiles = set()
 621         filesToChangeExecBit = {}
 622         for line in diff:
 623             diff = parseDiffTreeEntry(line)
 624             modifier = diff['status']
 625             path = diff['src']
 626             if modifier == "M":
 627                 p4_system("edit \"%s\"" % path)
 628                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                     filesToChangeExecBit[path] = diff['dst_mode']
 630                 editedFiles.add(path)
 631             elif modifier == "A":
 632                 filesToAdd.add(path)
 633                 filesToChangeExecBit[path] = diff['dst_mode']
 634                 if path in filesToDelete:
 635                     filesToDelete.remove(path)
 636             elif modifier == "D":
 637                 filesToDelete.add(path)
 638                 if path in filesToAdd:
 639                     filesToAdd.remove(path)
 640             elif modifier == "R":
 641                 src, dest = diff['src'], diff['dst']
 642                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                 p4_system("edit \"%s\"" % (dest))
 644                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                     filesToChangeExecBit[dest] = diff['dst_mode']
 646                 os.unlink(dest)
 647                 editedFiles.add(dest)
 648                 filesToDelete.add(src)
 649             else:
 650                 die("unknown modifier %s for %s" % (modifier, path))
 651
 652         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653         patchcmd = diffcmd + " | git apply "
 654         tryPatchCmd = patchcmd + "--check -"
 655         applyPatchCmd = patchcmd + "--check --apply -"
 656
 657         if os.system(tryPatchCmd) != 0:
 658             print "Unfortunately applying the change failed!"
 659             print "What do you want to do?"
 660             response = "x"
 661             while response != "s" and response != "a" and response != "w":
 662                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664             if response == "s":
 665                 print "Skipping! Good luck with the next patches..."
 666                 for f in editedFiles:
 667                     p4_system("revert \"%s\"" % f);
 668                 for f in filesToAdd:
 669                     system("rm %s" %f)
 670                 return
 671             elif response == "a":
 672                 os.system(applyPatchCmd)
 673                 if len(filesToAdd) > 0:
 674                     print "You may also want to call p4 add on the following files:"
 675                     print " ".join(filesToAdd)
 676                 if len(filesToDelete):
 677                     print "The following files should be scheduled for deletion with p4 delete:"
 678                     print " ".join(filesToDelete)
 679                 die("Please resolve and submit the conflict manually and "
 680                     + "continue afterwards with git-p4 submit --continue")
 681             elif response == "w":
 682                 system(diffcmd + " > patch.txt")
 683                 print "Patch saved to patch.txt in %s !" % self.clientPath
 684                 die("Please resolve and submit the conflict manually and "
 685                     "continue afterwards with git-p4 submit --continue")
 686
 687         system(applyPatchCmd)
 688
 689         for f in filesToAdd:
 690             p4_system("add \"%s\"" % f)
 691         for f in filesToDelete:
 692             p4_system("revert \"%s\"" % f)
 693             p4_system("delete \"%s\"" % f)
 694
 695         # Set/clear executable bits
 696         for f in filesToChangeExecBit.keys():
 697             mode = filesToChangeExecBit[f]
 698             setP4ExecBit(f, mode)
 699
 700         logMessage = extractLogMessageFromGitCommit(id)
 701         logMessage = logMessage.strip()
 702
 703         template = self.prepareSubmitTemplate()
 704
 705         if self.interactive:
 706             submitTemplate = self.prepareLogMessage(template, logMessage)
 707             if os.environ.has_key("P4DIFF"):
 708                 del(os.environ["P4DIFF"])
 709             diff = p4_read_pipe("diff -du ...")
 710
 711             newdiff = ""
 712             for newFile in filesToAdd:
 713                 newdiff += "==== new file ====\n"
 714                 newdiff += "--- /dev/null\n"
 715                 newdiff += "+++ %s\n" % newFile
 716                 f = open(newFile, "r")
 717                 for line in f.readlines():
 718                     newdiff += "+" + line
 719                 f.close()
 720
 721             separatorLine = "######## everything below this line is just the diff #######\n"
 722
 723             [handle, fileName] = tempfile.mkstemp()
 724             tmpFile = os.fdopen(handle, "w+")
 725             if self.isWindows:
 726                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 727                 separatorLine = separatorLine.replace("\n", "\r\n")
 728                 newdiff = newdiff.replace("\n", "\r\n")
 729             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 730             tmpFile.close()
 731             mtime = os.stat(fileName).st_mtime
 732             if os.environ.has_key("P4EDITOR"):
 733                 editor = os.environ.get("P4EDITOR")
 734             else:
 735                 editor = read_pipe("git var GIT_EDITOR").strip()
 736             system(editor + " " + fileName)
 737
 738             response = "y"
 739             if os.stat(fileName).st_mtime <= mtime:
 740                 response = "x"
 741                 while response != "y" and response != "n":
 742                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 743
 744             if response == "y":
 745                 tmpFile = open(fileName, "rb")
 746                 message = tmpFile.read()
 747                 tmpFile.close()
 748                 submitTemplate = message[:message.index(separatorLine)]
 749                 if self.isWindows:
 750                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 751                 p4_write_pipe("submit -i", submitTemplate)
 752             else:
 753                 for f in editedFiles:
 754                     p4_system("revert \"%s\"" % f);
 755                 for f in filesToAdd:
 756                     p4_system("revert \"%s\"" % f);
 757                     system("rm %s" %f)
 758
 759             os.remove(fileName)
 760         else:
 761             fileName = "submit.txt"
 762             file = open(fileName, "w+")
 763             file.write(self.prepareLogMessage(template, logMessage))
 764             file.close()
 765             print ("Perforce submit template written as %s. "
 766                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 767                    % (fileName, fileName))
 768
 769     def run(self, args):
 770         if len(args) == 0:
 771             self.master = currentGitBranch()
 772             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 773                 die("Detecting current git branch failed!")
 774         elif len(args) == 1:
 775             self.master = args[0]
 776         else:
 777             return False
 778
 779         allowSubmit = gitConfig("git-p4.allowSubmit")
 780         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 781             die("%s is not in git-p4.allowSubmit" % self.master)
 782
 783         [upstream, settings] = findUpstreamBranchPoint()
 784         self.depotPath = settings['depot-paths'][0]
 785         if len(self.origin) == 0:
 786             self.origin = upstream
 787
 788         if self.verbose:
 789             print "Origin branch is " + self.origin
 790
 791         if len(self.depotPath) == 0:
 792             print "Internal error: cannot locate perforce depot path from existing branches"
 793             sys.exit(128)
 794
 795         self.clientPath = p4Where(self.depotPath)
 796
 797         if len(self.clientPath) == 0:
 798             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 799             sys.exit(128)
 800
 801         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 802         self.oldWorkingDirectory = os.getcwd()
 803
 804         chdir(self.clientPath)
 805         print "Syncronizing p4 checkout..."
 806         p4_system("sync ...")
 807
 808         self.check()
 809
 810         commits = []
 811         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 812             commits.append(line.strip())
 813         commits.reverse()
 814
 815         while len(commits) > 0:
 816             commit = commits[0]
 817             commits = commits[1:]
 818             self.applyCommit(commit)
 819             if not self.interactive:
 820                 break
 821
 822         if len(commits) == 0:
 823             print "All changes applied!"
 824             chdir(self.oldWorkingDirectory)
 825
 826             sync = P4Sync()
 827             sync.run([])
 828
 829             rebase = P4Rebase()
 830             rebase.rebase()
 831
 832         return True
 833
 834 class P4Sync(Command):
 835     def __init__(self):
 836         Command.__init__(self)
 837         self.options = [
 838                 optparse.make_option("--branch", dest="branch"),
 839                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 840                 optparse.make_option("--changesfile", dest="changesFile"),
 841                 optparse.make_option("--silent", dest="silent", action="store_true"),
 842                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 843                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 844                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 845                                      help="Import into refs/heads/ , not refs/remotes"),
 846                 optparse.make_option("--max-changes", dest="maxChanges"),
 847                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 848                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 849                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 850                                      help="Only sync files that are included in the Perforce Client Spec")
 851         ]
 852         self.description = """Imports from Perforce into a git repository.\n
 853     example:
 854     //depot/my/project/ -- to import the current head
 855     //depot/my/project/@all -- to import everything
 856     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 857
 858     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 859
 860         self.usage += " //depot/path[@revRange]"
 861         self.silent = False
 862         self.createdBranches = set()
 863         self.committedChanges = set()
 864         self.branch = ""
 865         self.detectBranches = False
 866         self.detectLabels = False
 867         self.changesFile = ""
 868         self.syncWithOrigin = True
 869         self.verbose = False
 870         self.importIntoRemotes = True
 871         self.maxChanges = ""
 872         self.isWindows = (platform.system() == "Windows")
 873         self.keepRepoPath = False
 874         self.depotPaths = None
 875         self.p4BranchesInGit = []
 876         self.cloneExclude = []
 877         self.useClientSpec = False
 878         self.clientSpecDirs = []
 879
 880         if gitConfig("git-p4.syncFromOrigin") == "false":
 881             self.syncWithOrigin = False
 882
 883     def extractFilesFromCommit(self, commit):
 884         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 885                              for path in self.cloneExclude]
 886         files = []
 887         fnum = 0
 888         while commit.has_key("depotFile%s" % fnum):
 889             path =  commit["depotFile%s" % fnum]
 890
 891             if [p for p in self.cloneExclude
 892                 if path.startswith (p)]:
 893                 found = False
 894             else:
 895                 found = [p for p in self.depotPaths
 896                          if path.startswith (p)]
 897             if not found:
 898                 fnum = fnum + 1
 899                 continue
 900
 901             file = {}
 902             file["path"] = path
 903             file["rev"] = commit["rev%s" % fnum]
 904             file["action"] = commit["action%s" % fnum]
 905             file["type"] = commit["type%s" % fnum]
 906             files.append(file)
 907             fnum = fnum + 1
 908         return files
 909
 910     def stripRepoPath(self, path, prefixes):
 911         if self.keepRepoPath:
 912             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 913
 914         for p in prefixes:
 915             if path.startswith(p):
 916                 path = path[len(p):]
 917
 918         return path
 919
 920     def splitFilesIntoBranches(self, commit):
 921         branches = {}
 922         fnum = 0
 923         while commit.has_key("depotFile%s" % fnum):
 924             path =  commit["depotFile%s" % fnum]
 925             found = [p for p in self.depotPaths
 926                      if path.startswith (p)]
 927             if not found:
 928                 fnum = fnum + 1
 929                 continue
 930
 931             file = {}
 932             file["path"] = path
 933             file["rev"] = commit["rev%s" % fnum]
 934             file["action"] = commit["action%s" % fnum]
 935             file["type"] = commit["type%s" % fnum]
 936             fnum = fnum + 1
 937
 938             relPath = self.stripRepoPath(path, self.depotPaths)
 939
 940             for branch in self.knownBranches.keys():
 941
 942                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 943                 if relPath.startswith(branch + "/"):
 944                     if branch not in branches:
 945                         branches[branch] = []
 946                     branches[branch].append(file)
 947                     break
 948
 949         return branches
 950
 951     # output one file from the P4 stream
 952     # - helper for streamP4Files
 953
 954     def streamOneP4File(self, file, contents):
 955         if file["type"] == "apple":
 956             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 957                 file['depotFile']
 958             return
 959
 960         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 961         if verbose:
 962             sys.stderr.write("%s\n" % relPath)
 963
 964         mode = "644"
 965         if isP4Exec(file["type"]):
 966             mode = "755"
 967         elif file["type"] == "symlink":
 968             mode = "120000"
 969             # p4 print on a symlink contains "target\n", so strip it off
 970             data = ''.join(contents)
 971             contents = [data[:-1]]
 972
 973         if self.isWindows and file["type"].endswith("text"):
 974             mangled = []
 975             for data in contents:
 976                 data = data.replace("\r\n", "\n")
 977                 mangled.append(data)
 978             contents = mangled
 979
 980         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 981             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 982         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 983             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 984
 985         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 986
 987         # total length...
 988         length = 0
 989         for d in contents:
 990             length = length + len(d)
 991
 992         self.gitStream.write("data %d\n" % length)
 993         for d in contents:
 994             self.gitStream.write(d)
 995         self.gitStream.write("\n")
 996
 997     def streamOneP4Deletion(self, file):
 998         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
 999         if verbose:
1000             sys.stderr.write("delete %s\n" % relPath)
1001         self.gitStream.write("D %s\n" % relPath)
1002
1003     # handle another chunk of streaming data
1004     def streamP4FilesCb(self, marshalled):
1005
1006         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1007             # start of a new file - output the old one first
1008             self.streamOneP4File(self.stream_file, self.stream_contents)
1009             self.stream_file = {}
1010             self.stream_contents = []
1011             self.stream_have_file_info = False
1012
1013         # pick up the new file information... for the
1014         # 'data' field we need to append to our array
1015         for k in marshalled.keys():
1016             if k == 'data':
1017                 self.stream_contents.append(marshalled['data'])
1018             else:
1019                 self.stream_file[k] = marshalled[k]
1020
1021         self.stream_have_file_info = True
1022
1023     # Stream directly from "p4 files" into "git fast-import"
1024     def streamP4Files(self, files):
1025         filesForCommit = []
1026         filesToRead = []
1027         filesToDelete = []
1028
1029         for f in files:
1030             includeFile = True
1031             for val in self.clientSpecDirs:
1032                 if f['path'].startswith(val[0]):
1033                     if val[1] <= 0:
1034                         includeFile = False
1035                     break
1036
1037             if includeFile:
1038                 filesForCommit.append(f)
1039                 if f['action'] not in ('delete', 'move/delete', 'purge'):
1040                     filesToRead.append(f)
1041                 else:
1042                     filesToDelete.append(f)
1043
1044         # deleted files...
1045         for f in filesToDelete:
1046             self.streamOneP4Deletion(f)
1047
1048         if len(filesToRead) > 0:
1049             self.stream_file = {}
1050             self.stream_contents = []
1051             self.stream_have_file_info = False
1052
1053             # curry self argument
1054             def streamP4FilesCbSelf(entry):
1055                 self.streamP4FilesCb(entry)
1056
1057             p4CmdList("-x - print",
1058                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1059                                                   for f in filesToRead]),
1060                 cb=streamP4FilesCbSelf)
1061
1062             # do the last chunk
1063             if self.stream_file.has_key('depotFile'):
1064                 self.streamOneP4File(self.stream_file, self.stream_contents)
1065
1066     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1067         epoch = details["time"]
1068         author = details["user"]
1069         self.branchPrefixes = branchPrefixes
1070
1071         if self.verbose:
1072             print "commit into %s" % branch
1073
1074         # start with reading files; if that fails, we should not
1075         # create a commit.
1076         new_files = []
1077         for f in files:
1078             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1079                 new_files.append (f)
1080             else:
1081                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1082
1083         self.gitStream.write("commit %s\n" % branch)
1084 #        gitStream.write("mark :%s\n" % details["change"])
1085         self.committedChanges.add(int(details["change"]))
1086         committer = ""
1087         if author not in self.users:
1088             self.getUserMapFromPerforceServer()
1089         if author in self.users:
1090             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1091         else:
1092             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1093
1094         self.gitStream.write("committer %s\n" % committer)
1095
1096         self.gitStream.write("data <<EOT\n")
1097         self.gitStream.write(details["desc"])
1098         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1099                              % (','.join (branchPrefixes), details["change"]))
1100         if len(details['options']) > 0:
1101             self.gitStream.write(": options = %s" % details['options'])
1102         self.gitStream.write("]\nEOT\n\n")
1103
1104         if len(parent) > 0:
1105             if self.verbose:
1106                 print "parent %s" % parent
1107             self.gitStream.write("from %s\n" % parent)
1108
1109         self.streamP4Files(new_files)
1110         self.gitStream.write("\n")
1111
1112         change = int(details["change"])
1113
1114         if self.labels.has_key(change):
1115             label = self.labels[change]
1116             labelDetails = label[0]
1117             labelRevisions = label[1]
1118             if self.verbose:
1119                 print "Change %s is labelled %s" % (change, labelDetails)
1120
1121             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1122                                                     for p in branchPrefixes]))
1123
1124             if len(files) == len(labelRevisions):
1125
1126                 cleanedFiles = {}
1127                 for info in files:
1128                     if info["action"] in ("delete", "purge"):
1129                         continue
1130                     cleanedFiles[info["depotFile"]] = info["rev"]
1131
1132                 if cleanedFiles == labelRevisions:
1133                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1134                     self.gitStream.write("from %s\n" % branch)
1135
1136                     owner = labelDetails["Owner"]
1137                     tagger = ""
1138                     if author in self.users:
1139                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1140                     else:
1141                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1142                     self.gitStream.write("tagger %s\n" % tagger)
1143                     self.gitStream.write("data <<EOT\n")
1144                     self.gitStream.write(labelDetails["Description"])
1145                     self.gitStream.write("EOT\n\n")
1146
1147                 else:
1148                     if not self.silent:
1149                         print ("Tag %s does not match with change %s: files do not match."
1150                                % (labelDetails["label"], change))
1151
1152             else:
1153                 if not self.silent:
1154                     print ("Tag %s does not match with change %s: file count is different."
1155                            % (labelDetails["label"], change))
1156
1157     def getUserCacheFilename(self):
1158         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1159         return home + "/.gitp4-usercache.txt"
1160
1161     def getUserMapFromPerforceServer(self):
1162         if self.userMapFromPerforceServer:
1163             return
1164         self.users = {}
1165
1166         for output in p4CmdList("users"):
1167             if not output.has_key("User"):
1168                 continue
1169             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1170
1171
1172         s = ''
1173         for (key, val) in self.users.items():
1174             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1175
1176         open(self.getUserCacheFilename(), "wb").write(s)
1177         self.userMapFromPerforceServer = True
1178
1179     def loadUserMapFromCache(self):
1180         self.users = {}
1181         self.userMapFromPerforceServer = False
1182         try:
1183             cache = open(self.getUserCacheFilename(), "rb")
1184             lines = cache.readlines()
1185             cache.close()
1186             for line in lines:
1187                 entry = line.strip().split("\t")
1188                 self.users[entry[0]] = entry[1]
1189         except IOError:
1190             self.getUserMapFromPerforceServer()
1191
1192     def getLabels(self):
1193         self.labels = {}
1194
1195         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1196         if len(l) > 0 and not self.silent:
1197             print "Finding files belonging to labels in %s" % `self.depotPaths`
1198
1199         for output in l:
1200             label = output["label"]
1201             revisions = {}
1202             newestChange = 0
1203             if self.verbose:
1204                 print "Querying files for label %s" % label
1205             for file in p4CmdList("files "
1206                                   +  ' '.join (["%s...@%s" % (p, label)
1207                                                 for p in self.depotPaths])):
1208                 revisions[file["depotFile"]] = file["rev"]
1209                 change = int(file["change"])
1210                 if change > newestChange:
1211                     newestChange = change
1212
1213             self.labels[newestChange] = [output, revisions]
1214
1215         if self.verbose:
1216             print "Label changes: %s" % self.labels.keys()
1217
1218     def guessProjectName(self):
1219         for p in self.depotPaths:
1220             if p.endswith("/"):
1221                 p = p[:-1]
1222             p = p[p.strip().rfind("/") + 1:]
1223             if not p.endswith("/"):
1224                p += "/"
1225             return p
1226
1227     def getBranchMapping(self):
1228         lostAndFoundBranches = set()
1229
1230         for info in p4CmdList("branches"):
1231             details = p4Cmd("branch -o %s" % info["branch"])
1232             viewIdx = 0
1233             while details.has_key("View%s" % viewIdx):
1234                 paths = details["View%s" % viewIdx].split(" ")
1235                 viewIdx = viewIdx + 1
1236                 # require standard //depot/foo/... //depot/bar/... mapping
1237                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1238                     continue
1239                 source = paths[0]
1240                 destination = paths[1]
1241                 ## HACK
1242                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1243                     source = source[len(self.depotPaths[0]):-4]
1244                     destination = destination[len(self.depotPaths[0]):-4]
1245
1246                     if destination in self.knownBranches:
1247                         if not self.silent:
1248                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1249                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1250                         continue
1251
1252                     self.knownBranches[destination] = source
1253
1254                     lostAndFoundBranches.discard(destination)
1255
1256                     if source not in self.knownBranches:
1257                         lostAndFoundBranches.add(source)
1258
1259
1260         for branch in lostAndFoundBranches:
1261             self.knownBranches[branch] = branch
1262
1263     def getBranchMappingFromGitBranches(self):
1264         branches = p4BranchesInGit(self.importIntoRemotes)
1265         for branch in branches.keys():
1266             if branch == "master":
1267                 branch = "main"
1268             else:
1269                 branch = branch[len(self.projectName):]
1270             self.knownBranches[branch] = branch
1271
1272     def listExistingP4GitBranches(self):
1273         # branches holds mapping from name to commit
1274         branches = p4BranchesInGit(self.importIntoRemotes)
1275         self.p4BranchesInGit = branches.keys()
1276         for branch in branches.keys():
1277             self.initialParents[self.refPrefix + branch] = branches[branch]
1278
1279     def updateOptionDict(self, d):
1280         option_keys = {}
1281         if self.keepRepoPath:
1282             option_keys['keepRepoPath'] = 1
1283
1284         d["options"] = ' '.join(sorted(option_keys.keys()))
1285
1286     def readOptions(self, d):
1287         self.keepRepoPath = (d.has_key('options')
1288                              and ('keepRepoPath' in d['options']))
1289
1290     def gitRefForBranch(self, branch):
1291         if branch == "main":
1292             return self.refPrefix + "master"
1293
1294         if len(branch) <= 0:
1295             return branch
1296
1297         return self.refPrefix + self.projectName + branch
1298
1299     def gitCommitByP4Change(self, ref, change):
1300         if self.verbose:
1301             print "looking in ref " + ref + " for change %s using bisect..." % change
1302
1303         earliestCommit = ""
1304         latestCommit = parseRevision(ref)
1305
1306         while True:
1307             if self.verbose:
1308                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1309             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1310             if len(next) == 0:
1311                 if self.verbose:
1312                     print "argh"
1313                 return ""
1314             log = extractLogMessageFromGitCommit(next)
1315             settings = extractSettingsGitLog(log)
1316             currentChange = int(settings['change'])
1317             if self.verbose:
1318                 print "current change %s" % currentChange
1319
1320             if currentChange == change:
1321                 if self.verbose:
1322                     print "found %s" % next
1323                 return next
1324
1325             if currentChange < change:
1326                 earliestCommit = "^%s" % next
1327             else:
1328                 latestCommit = "%s" % next
1329
1330         return ""
1331
1332     def importNewBranch(self, branch, maxChange):
1333         # make fast-import flush all changes to disk and update the refs using the checkpoint
1334         # command so that we can try to find the branch parent in the git history
1335         self.gitStream.write("checkpoint\n\n");
1336         self.gitStream.flush();
1337         branchPrefix = self.depotPaths[0] + branch + "/"
1338         range = "@1,%s" % maxChange
1339         #print "prefix" + branchPrefix
1340         changes = p4ChangesForPaths([branchPrefix], range)
1341         if len(changes) <= 0:
1342             return False
1343         firstChange = changes[0]
1344         #print "first change in branch: %s" % firstChange
1345         sourceBranch = self.knownBranches[branch]
1346         sourceDepotPath = self.depotPaths[0] + sourceBranch
1347         sourceRef = self.gitRefForBranch(sourceBranch)
1348         #print "source " + sourceBranch
1349
1350         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1351         #print "branch parent: %s" % branchParentChange
1352         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1353         if len(gitParent) > 0:
1354             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1355             #print "parent git commit: %s" % gitParent
1356
1357         self.importChanges(changes)
1358         return True
1359
1360     def importChanges(self, changes):
1361         cnt = 1
1362         for change in changes:
1363             description = p4Cmd("describe %s" % change)
1364             self.updateOptionDict(description)
1365
1366             if not self.silent:
1367                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1368                 sys.stdout.flush()
1369             cnt = cnt + 1
1370
1371             try:
1372                 if self.detectBranches:
1373                     branches = self.splitFilesIntoBranches(description)
1374                     for branch in branches.keys():
1375                         ## HACK  --hwn
1376                         branchPrefix = self.depotPaths[0] + branch + "/"
1377
1378                         parent = ""
1379
1380                         filesForCommit = branches[branch]
1381
1382                         if self.verbose:
1383                             print "branch is %s" % branch
1384
1385                         self.updatedBranches.add(branch)
1386
1387                         if branch not in self.createdBranches:
1388                             self.createdBranches.add(branch)
1389                             parent = self.knownBranches[branch]
1390                             if parent == branch:
1391                                 parent = ""
1392                             else:
1393                                 fullBranch = self.projectName + branch
1394                                 if fullBranch not in self.p4BranchesInGit:
1395                                     if not self.silent:
1396                                         print("\n    Importing new branch %s" % fullBranch);
1397                                     if self.importNewBranch(branch, change - 1):
1398                                         parent = ""
1399                                         self.p4BranchesInGit.append(fullBranch)
1400                                     if not self.silent:
1401                                         print("\n    Resuming with change %s" % change);
1402
1403                                 if self.verbose:
1404                                     print "parent determined through known branches: %s" % parent
1405
1406                         branch = self.gitRefForBranch(branch)
1407                         parent = self.gitRefForBranch(parent)
1408
1409                         if self.verbose:
1410                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1411
1412                         if len(parent) == 0 and branch in self.initialParents:
1413                             parent = self.initialParents[branch]
1414                             del self.initialParents[branch]
1415
1416                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1417                 else:
1418                     files = self.extractFilesFromCommit(description)
1419                     self.commit(description, files, self.branch, self.depotPaths,
1420                                 self.initialParent)
1421                     self.initialParent = ""
1422             except IOError:
1423                 print self.gitError.read()
1424                 sys.exit(1)
1425
1426     def importHeadRevision(self, revision):
1427         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1428
1429         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1430         details["desc"] = ("Initial import of %s from the state at revision %s"
1431                            % (' '.join(self.depotPaths), revision))
1432         details["change"] = revision
1433         newestRevision = 0
1434
1435         fileCnt = 0
1436         for info in p4CmdList("files "
1437                               +  ' '.join(["%s...%s"
1438                                            % (p, revision)
1439                                            for p in self.depotPaths])):
1440
1441             if info['code'] == 'error':
1442                 sys.stderr.write("p4 returned an error: %s\n"
1443                                  % info['data'])
1444                 sys.exit(1)
1445
1446
1447             change = int(info["change"])
1448             if change > newestRevision:
1449                 newestRevision = change
1450
1451             if info["action"] in ("delete", "purge"):
1452                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1453                 #fileCnt = fileCnt + 1
1454                 continue
1455
1456             for prop in ["depotFile", "rev", "action", "type" ]:
1457                 details["%s%s" % (prop, fileCnt)] = info[prop]
1458
1459             fileCnt = fileCnt + 1
1460
1461         details["change"] = newestRevision
1462         self.updateOptionDict(details)
1463         try:
1464             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1465         except IOError:
1466             print "IO error with git fast-import. Is your git version recent enough?"
1467             print self.gitError.read()
1468
1469
1470     def getClientSpec(self):
1471         specList = p4CmdList( "client -o" )
1472         temp = {}
1473         for entry in specList:
1474             for k,v in entry.iteritems():
1475                 if k.startswith("View"):
1476                     if v.startswith('"'):
1477                         start = 1
1478                     else:
1479                         start = 0
1480                     index = v.find("...")
1481                     v = v[start:index]
1482                     if v.startswith("-"):
1483                         v = v[1:]
1484                         temp[v] = -len(v)
1485                     else:
1486                         temp[v] = len(v)
1487         self.clientSpecDirs = temp.items()
1488         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1489
1490     def run(self, args):
1491         self.depotPaths = []
1492         self.changeRange = ""
1493         self.initialParent = ""
1494         self.previousDepotPaths = []
1495
1496         # map from branch depot path to parent branch
1497         self.knownBranches = {}
1498         self.initialParents = {}
1499         self.hasOrigin = originP4BranchesExist()
1500         if not self.syncWithOrigin:
1501             self.hasOrigin = False
1502
1503         if self.importIntoRemotes:
1504             self.refPrefix = "refs/remotes/p4/"
1505         else:
1506             self.refPrefix = "refs/heads/p4/"
1507
1508         if self.syncWithOrigin and self.hasOrigin:
1509             if not self.silent:
1510                 print "Syncing with origin first by calling git fetch origin"
1511             system("git fetch origin")
1512
1513         if len(self.branch) == 0:
1514             self.branch = self.refPrefix + "master"
1515             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1516                 system("git update-ref %s refs/heads/p4" % self.branch)
1517                 system("git branch -D p4");
1518             # create it /after/ importing, when master exists
1519             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1520                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1521
1522         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1523             self.getClientSpec()
1524
1525         # TODO: should always look at previous commits,
1526         # merge with previous imports, if possible.
1527         if args == []:
1528             if self.hasOrigin:
1529                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1530             self.listExistingP4GitBranches()
1531
1532             if len(self.p4BranchesInGit) > 1:
1533                 if not self.silent:
1534                     print "Importing from/into multiple branches"
1535                 self.detectBranches = True
1536
1537             if self.verbose:
1538                 print "branches: %s" % self.p4BranchesInGit
1539
1540             p4Change = 0
1541             for branch in self.p4BranchesInGit:
1542                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1543
1544                 settings = extractSettingsGitLog(logMsg)
1545
1546                 self.readOptions(settings)
1547                 if (settings.has_key('depot-paths')
1548                     and settings.has_key ('change')):
1549                     change = int(settings['change']) + 1
1550                     p4Change = max(p4Change, change)
1551
1552                     depotPaths = sorted(settings['depot-paths'])
1553                     if self.previousDepotPaths == []:
1554                         self.previousDepotPaths = depotPaths
1555                     else:
1556                         paths = []
1557                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1558                             for i in range(0, min(len(cur), len(prev))):
1559                                 if cur[i] <> prev[i]:
1560                                     i = i - 1
1561                                     break
1562
1563                             paths.append (cur[:i + 1])
1564
1565                         self.previousDepotPaths = paths
1566
1567             if p4Change > 0:
1568                 self.depotPaths = sorted(self.previousDepotPaths)
1569                 self.changeRange = "@%s,#head" % p4Change
1570                 if not self.detectBranches:
1571                     self.initialParent = parseRevision(self.branch)
1572                 if not self.silent and not self.detectBranches:
1573                     print "Performing incremental import into %s git branch" % self.branch
1574
1575         if not self.branch.startswith("refs/"):
1576             self.branch = "refs/heads/" + self.branch
1577
1578         if len(args) == 0 and self.depotPaths:
1579             if not self.silent:
1580                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1581         else:
1582             if self.depotPaths and self.depotPaths != args:
1583                 print ("previous import used depot path %s and now %s was specified. "
1584                        "This doesn't work!" % (' '.join (self.depotPaths),
1585                                                ' '.join (args)))
1586                 sys.exit(1)
1587
1588             self.depotPaths = sorted(args)
1589
1590         revision = ""
1591         self.users = {}
1592
1593         newPaths = []
1594         for p in self.depotPaths:
1595             if p.find("@") != -1:
1596                 atIdx = p.index("@")
1597                 self.changeRange = p[atIdx:]
1598                 if self.changeRange == "@all":
1599                     self.changeRange = ""
1600                 elif ',' not in self.changeRange:
1601                     revision = self.changeRange
1602                     self.changeRange = ""
1603                 p = p[:atIdx]
1604             elif p.find("#") != -1:
1605                 hashIdx = p.index("#")
1606                 revision = p[hashIdx:]
1607                 p = p[:hashIdx]
1608             elif self.previousDepotPaths == []:
1609                 revision = "#head"
1610
1611             p = re.sub ("\.\.\.$", "", p)
1612             if not p.endswith("/"):
1613                 p += "/"
1614
1615             newPaths.append(p)
1616
1617         self.depotPaths = newPaths
1618
1619
1620         self.loadUserMapFromCache()
1621         self.labels = {}
1622         if self.detectLabels:
1623             self.getLabels();
1624
1625         if self.detectBranches:
1626             ## FIXME - what's a P4 projectName ?
1627             self.projectName = self.guessProjectName()
1628
1629             if self.hasOrigin:
1630                 self.getBranchMappingFromGitBranches()
1631             else:
1632                 self.getBranchMapping()
1633             if self.verbose:
1634                 print "p4-git branches: %s" % self.p4BranchesInGit
1635                 print "initial parents: %s" % self.initialParents
1636             for b in self.p4BranchesInGit:
1637                 if b != "master":
1638
1639                     ## FIXME
1640                     b = b[len(self.projectName):]
1641                 self.createdBranches.add(b)
1642
1643         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1644
1645         importProcess = subprocess.Popen(["git", "fast-import"],
1646                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1647                                          stderr=subprocess.PIPE);
1648         self.gitOutput = importProcess.stdout
1649         self.gitStream = importProcess.stdin
1650         self.gitError = importProcess.stderr
1651
1652         if revision:
1653             self.importHeadRevision(revision)
1654         else:
1655             changes = []
1656
1657             if len(self.changesFile) > 0:
1658                 output = open(self.changesFile).readlines()
1659                 changeSet = set()
1660                 for line in output:
1661                     changeSet.add(int(line))
1662
1663                 for change in changeSet:
1664                     changes.append(change)
1665
1666                 changes.sort()
1667             else:
1668                 if self.verbose:
1669                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1670                                                               self.changeRange)
1671                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1672
1673                 if len(self.maxChanges) > 0:
1674                     changes = changes[:min(int(self.maxChanges), len(changes))]
1675
1676             if len(changes) == 0:
1677                 if not self.silent:
1678                     print "No changes to import!"
1679                 return True
1680
1681             if not self.silent and not self.detectBranches:
1682                 print "Import destination: %s" % self.branch
1683
1684             self.updatedBranches = set()
1685
1686             self.importChanges(changes)
1687
1688             if not self.silent:
1689                 print ""
1690                 if len(self.updatedBranches) > 0:
1691                     sys.stdout.write("Updated branches: ")
1692                     for b in self.updatedBranches:
1693                         sys.stdout.write("%s " % b)
1694                     sys.stdout.write("\n")
1695
1696         self.gitStream.close()
1697         if importProcess.wait() != 0:
1698             die("fast-import failed: %s" % self.gitError.read())
1699         self.gitOutput.close()
1700         self.gitError.close()
1701
1702         return True
1703
1704 class P4Rebase(Command):
1705     def __init__(self):
1706         Command.__init__(self)
1707         self.options = [ ]
1708         self.description = ("Fetches the latest revision from perforce and "
1709                             + "rebases the current work (branch) against it")
1710         self.verbose = False
1711
1712     def run(self, args):
1713         sync = P4Sync()
1714         sync.run([])
1715
1716         return self.rebase()
1717
1718     def rebase(self):
1719         if os.system("git update-index --refresh") != 0:
1720             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1721         if len(read_pipe("git diff-index HEAD --")) > 0:
1722             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1723
1724         [upstream, settings] = findUpstreamBranchPoint()
1725         if len(upstream) == 0:
1726             die("Cannot find upstream branchpoint for rebase")
1727
1728         # the branchpoint may be p4/foo~3, so strip off the parent
1729         upstream = re.sub("~[0-9]+$", "", upstream)
1730
1731         print "Rebasing the current branch onto %s" % upstream
1732         oldHead = read_pipe("git rev-parse HEAD").strip()
1733         system("git rebase %s" % upstream)
1734         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1735         return True
1736
1737 class P4Clone(P4Sync):
1738     def __init__(self):
1739         P4Sync.__init__(self)
1740         self.description = "Creates a new git repository and imports from Perforce into it"
1741         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1742         self.options += [
1743             optparse.make_option("--destination", dest="cloneDestination",
1744                                  action='store', default=None,
1745                                  help="where to leave result of the clone"),
1746             optparse.make_option("-/", dest="cloneExclude",
1747                                  action="append", type="string",
1748                                  help="exclude depot path")
1749         ]
1750         self.cloneDestination = None
1751         self.needsGit = False
1752
1753     # This is required for the "append" cloneExclude action
1754     def ensure_value(self, attr, value):
1755         if not hasattr(self, attr) or getattr(self, attr) is None:
1756             setattr(self, attr, value)
1757         return getattr(self, attr)
1758
1759     def defaultDestination(self, args):
1760         ## TODO: use common prefix of args?
1761         depotPath = args[0]
1762         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1763         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1764         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1765         depotDir = re.sub(r"/$", "", depotDir)
1766         return os.path.split(depotDir)[1]
1767
1768     def run(self, args):
1769         if len(args) < 1:
1770             return False
1771
1772         if self.keepRepoPath and not self.cloneDestination:
1773             sys.stderr.write("Must specify destination for --keep-path\n")
1774             sys.exit(1)
1775
1776         depotPaths = args
1777
1778         if not self.cloneDestination and len(depotPaths) > 1:
1779             self.cloneDestination = depotPaths[-1]
1780             depotPaths = depotPaths[:-1]
1781
1782         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1783         for p in depotPaths:
1784             if not p.startswith("//"):
1785                 return False
1786
1787         if not self.cloneDestination:
1788             self.cloneDestination = self.defaultDestination(args)
1789
1790         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1791         if not os.path.exists(self.cloneDestination):
1792             os.makedirs(self.cloneDestination)
1793         chdir(self.cloneDestination)
1794         system("git init")
1795         self.gitdir = os.getcwd() + "/.git"
1796         if not P4Sync.run(self, depotPaths):
1797             return False
1798         if self.branch != "master":
1799             if self.importIntoRemotes:
1800                 masterbranch = "refs/remotes/p4/master"
1801             else:
1802                 masterbranch = "refs/heads/p4/master"
1803             if gitBranchExists(masterbranch):
1804                 system("git branch master %s" % masterbranch)
1805                 system("git checkout -f")
1806             else:
1807                 print "Could not detect main branch. No checkout/master branch created."
1808
1809         return True
1810
1811 class P4Branches(Command):
1812     def __init__(self):
1813         Command.__init__(self)
1814         self.options = [ ]
1815         self.description = ("Shows the git branches that hold imports and their "
1816                             + "corresponding perforce depot paths")
1817         self.verbose = False
1818
1819     def run(self, args):
1820         if originP4BranchesExist():
1821             createOrUpdateBranchesFromOrigin()
1822
1823         cmdline = "git rev-parse --symbolic "
1824         cmdline += " --remotes"
1825
1826         for line in read_pipe_lines(cmdline):
1827             line = line.strip()
1828
1829             if not line.startswith('p4/') or line == "p4/HEAD":
1830                 continue
1831             branch = line
1832
1833             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1834             settings = extractSettingsGitLog(log)
1835
1836             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1837         return True
1838
1839 class HelpFormatter(optparse.IndentedHelpFormatter):
1840     def __init__(self):
1841         optparse.IndentedHelpFormatter.__init__(self)
1842
1843     def format_description(self, description):
1844         if description:
1845             return description + "\n"
1846         else:
1847             return ""
1848
1849 def printUsage(commands):
1850     print "usage: %s <command> [options]" % sys.argv[0]
1851     print ""
1852     print "valid commands: %s" % ", ".join(commands)
1853     print ""
1854     print "Try %s <command> --help for command specific help." % sys.argv[0]
1855     print ""
1856
1857 commands = {
1858     "debug" : P4Debug,
1859     "submit" : P4Submit,
1860     "commit" : P4Submit,
1861     "sync" : P4Sync,
1862     "rebase" : P4Rebase,
1863     "clone" : P4Clone,
1864     "rollback" : P4RollBack,
1865     "branches" : P4Branches
1866 }
1867
1868
1869 def main():
1870     if len(sys.argv[1:]) == 0:
1871         printUsage(commands.keys())
1872         sys.exit(2)
1873
1874     cmd = ""
1875     cmdName = sys.argv[1]
1876     try:
1877         klass = commands[cmdName]
1878         cmd = klass()
1879     except KeyError:
1880         print "unknown command %s" % cmdName
1881         print ""
1882         printUsage(commands.keys())
1883         sys.exit(2)
1884
1885     options = cmd.options
1886     cmd.gitdir = os.environ.get("GIT_DIR", None)
1887
1888     args = sys.argv[2:]
1889
1890     if len(options) > 0:
1891         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1892
1893         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1894                                        options,
1895                                        description = cmd.description,
1896                                        formatter = HelpFormatter())
1897
1898         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1899     global verbose
1900     verbose = cmd.verbose
1901     if cmd.needsGit:
1902         if cmd.gitdir == None:
1903             cmd.gitdir = os.path.abspath(".git")
1904             if not isValidGitDir(cmd.gitdir):
1905                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1906                 if os.path.exists(cmd.gitdir):
1907                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1908                     if len(cdup) > 0:
1909                         chdir(cdup);
1910
1911         if not isValidGitDir(cmd.gitdir):
1912             if isValidGitDir(cmd.gitdir + "/.git"):
1913                 cmd.gitdir += "/.git"
1914             else:
1915                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1916
1917         os.environ["GIT_DIR"] = cmd.gitdir
1918
1919     if not cmd.run(args):
1920         parser.print_help()
1921
1922
1923 if __name__ == '__main__':
1924     main()