git.oblomov.eu Git - git/blob - contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.
  57     os.environ['PWD']=dir
  58     os.chdir(dir)
  59
  60 def die(msg):
  61     if verbose:
  62         raise Exception(msg)
  63     else:
  64         sys.stderr.write(msg + "\n")
  65         sys.exit(1)
  66
  67 def write_pipe(c, stdin):
  68     if verbose:
  69         sys.stderr.write('Writing pipe: %s\n' % str(c))
  70
  71     expand = isinstance(c,basestring)
  72     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  73     pipe = p.stdin
  74     val = pipe.write(stdin)
  75     pipe.close()
  76     if p.wait():
  77         die('Command failed: %s' % str(c))
  78
  79     return val
  80
  81 def p4_write_pipe(c, stdin):
  82     real_cmd = p4_build_cmd(c)
  83     return write_pipe(real_cmd, stdin)
  84
  85 def read_pipe(c, ignore_error=False):
  86     if verbose:
  87         sys.stderr.write('Reading pipe: %s\n' % str(c))
  88
  89     expand = isinstance(c,basestring)
  90     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  91     pipe = p.stdout
  92     val = pipe.read()
  93     if p.wait() and not ignore_error:
  94         die('Command failed: %s' % str(c))
  95
  96     return val
  97
  98 def p4_read_pipe(c, ignore_error=False):
  99     real_cmd = p4_build_cmd(c)
 100     return read_pipe(real_cmd, ignore_error)
 101
 102 def read_pipe_lines(c):
 103     if verbose:
 104         sys.stderr.write('Reading pipe: %s\n' % str(c))
 105
 106     expand = isinstance(c, basestring)
 107     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 108     pipe = p.stdout
 109     val = pipe.readlines()
 110     if pipe.close() or p.wait():
 111         die('Command failed: %s' % str(c))
 112
 113     return val
 114
 115 def p4_read_pipe_lines(c):
 116     """Specifically invoke p4 on the command supplied. """
 117     real_cmd = p4_build_cmd(c)
 118     return read_pipe_lines(real_cmd)
 119
 120 def system(cmd):
 121     expand = isinstance(cmd,basestring)
 122     if verbose:
 123         sys.stderr.write("executing %s\n" % str(cmd))
 124     subprocess.check_call(cmd, shell=expand)
 125
 126 def p4_system(cmd):
 127     """Specifically invoke p4 as the system command. """
 128     real_cmd = p4_build_cmd(cmd)
 129     expand = isinstance(real_cmd, basestring)
 130     subprocess.check_call(real_cmd, shell=expand)
 131
 132 def p4_integrate(src, dest):
 133     p4_system(["integrate", "-Dt", src, dest])
 134
 135 def p4_sync(path):
 136     p4_system(["sync", path])
 137
 138 def p4_add(f):
 139     p4_system(["add", f])
 140
 141 def p4_delete(f):
 142     p4_system(["delete", f])
 143
 144 def p4_edit(f):
 145     p4_system(["edit", f])
 146
 147 def p4_revert(f):
 148     p4_system(["revert", f])
 149
 150 def p4_reopen(type, file):
 151     p4_system(["reopen", "-t", type, file])
 152
 153 #
 154 # Canonicalize the p4 type and return a tuple of the
 155 # base type, plus any modifiers.  See "p4 help filetypes"
 156 # for a list and explanation.
 157 #
 158 def split_p4_type(p4type):
 159
 160     p4_filetypes_historical = {
 161         "ctempobj": "binary+Sw",
 162         "ctext": "text+C",
 163         "cxtext": "text+Cx",
 164         "ktext": "text+k",
 165         "kxtext": "text+kx",
 166         "ltext": "text+F",
 167         "tempobj": "binary+FSw",
 168         "ubinary": "binary+F",
 169         "uresource": "resource+F",
 170         "uxbinary": "binary+Fx",
 171         "xbinary": "binary+x",
 172         "xltext": "text+Fx",
 173         "xtempobj": "binary+Swx",
 174         "xtext": "text+x",
 175         "xunicode": "unicode+x",
 176         "xutf16": "utf16+x",
 177     }
 178     if p4type in p4_filetypes_historical:
 179         p4type = p4_filetypes_historical[p4type]
 180     mods = ""
 181     s = p4type.split("+")
 182     base = s[0]
 183     mods = ""
 184     if len(s) > 1:
 185         mods = s[1]
 186     return (base, mods)
 187
 188
 189 def setP4ExecBit(file, mode):
 190     # Reopens an already open file and changes the execute bit to match
 191     # the execute bit setting in the passed in mode.
 192
 193     p4Type = "+x"
 194
 195     if not isModeExec(mode):
 196         p4Type = getP4OpenedType(file)
 197         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 198         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 199         if p4Type[-1] == "+":
 200             p4Type = p4Type[0:-1]
 201
 202     p4_reopen(p4Type, file)
 203
 204 def getP4OpenedType(file):
 205     # Returns the perforce file type for the given file.
 206
 207     result = p4_read_pipe(["opened", file])
 208     match = re.match(".*\((.+)\)\r?$", result)
 209     if match:
 210         return match.group(1)
 211     else:
 212         die("Could not determine file type for %s (result: '%s')" % (file, result))
 213
 214 def diffTreePattern():
 215     # This is a simple generator for the diff tree regex pattern. This could be
 216     # a class variable if this and parseDiffTreeEntry were a part of a class.
 217     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 218     while True:
 219         yield pattern
 220
 221 def parseDiffTreeEntry(entry):
 222     """Parses a single diff tree entry into its component elements.
 223
 224     See git-diff-tree(1) manpage for details about the format of the diff
 225     output. This method returns a dictionary with the following elements:
 226
 227     src_mode - The mode of the source file
 228     dst_mode - The mode of the destination file
 229     src_sha1 - The sha1 for the source file
 230     dst_sha1 - The sha1 fr the destination file
 231     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 232     status_score - The score for the status (applicable for 'C' and 'R'
 233                    statuses). This is None if there is no score.
 234     src - The path for the source file.
 235     dst - The path for the destination file. This is only present for
 236           copy or renames. If it is not present, this is None.
 237
 238     If the pattern is not matched, None is returned."""
 239
 240     match = diffTreePattern().next().match(entry)
 241     if match:
 242         return {
 243             'src_mode': match.group(1),
 244             'dst_mode': match.group(2),
 245             'src_sha1': match.group(3),
 246             'dst_sha1': match.group(4),
 247             'status': match.group(5),
 248             'status_score': match.group(6),
 249             'src': match.group(7),
 250             'dst': match.group(10)
 251         }
 252     return None
 253
 254 def isModeExec(mode):
 255     # Returns True if the given git mode represents an executable file,
 256     # otherwise False.
 257     return mode[-3:] == "755"
 258
 259 def isModeExecChanged(src_mode, dst_mode):
 260     return isModeExec(src_mode) != isModeExec(dst_mode)
 261
 262 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 263
 264     if isinstance(cmd,basestring):
 265         cmd = "-G " + cmd
 266         expand = True
 267     else:
 268         cmd = ["-G"] + cmd
 269         expand = False
 270
 271     cmd = p4_build_cmd(cmd)
 272     if verbose:
 273         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 274
 275     # Use a temporary file to avoid deadlocks without
 276     # subprocess.communicate(), which would put another copy
 277     # of stdout into memory.
 278     stdin_file = None
 279     if stdin is not None:
 280         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 281         if isinstance(stdin,basestring):
 282             stdin_file.write(stdin)
 283         else:
 284             for i in stdin:
 285                 stdin_file.write(i + '\n')
 286         stdin_file.flush()
 287         stdin_file.seek(0)
 288
 289     p4 = subprocess.Popen(cmd,
 290                           shell=expand,
 291                           stdin=stdin_file,
 292                           stdout=subprocess.PIPE)
 293
 294     result = []
 295     try:
 296         while True:
 297             entry = marshal.load(p4.stdout)
 298             if cb is not None:
 299                 cb(entry)
 300             else:
 301                 result.append(entry)
 302     except EOFError:
 303         pass
 304     exitCode = p4.wait()
 305     if exitCode != 0:
 306         entry = {}
 307         entry["p4ExitCode"] = exitCode
 308         result.append(entry)
 309
 310     return result
 311
 312 def p4Cmd(cmd):
 313     list = p4CmdList(cmd)
 314     result = {}
 315     for entry in list:
 316         result.update(entry)
 317     return result;
 318
 319 def p4Where(depotPath):
 320     if not depotPath.endswith("/"):
 321         depotPath += "/"
 322     depotPath = depotPath + "..."
 323     outputList = p4CmdList(["where", depotPath])
 324     output = None
 325     for entry in outputList:
 326         if "depotFile" in entry:
 327             if entry["depotFile"] == depotPath:
 328                 output = entry
 329                 break
 330         elif "data" in entry:
 331             data = entry.get("data")
 332             space = data.find(" ")
 333             if data[:space] == depotPath:
 334                 output = entry
 335                 break
 336     if output == None:
 337         return ""
 338     if output["code"] == "error":
 339         return ""
 340     clientPath = ""
 341     if "path" in output:
 342         clientPath = output.get("path")
 343     elif "data" in output:
 344         data = output.get("data")
 345         lastSpace = data.rfind(" ")
 346         clientPath = data[lastSpace + 1:]
 347
 348     if clientPath.endswith("..."):
 349         clientPath = clientPath[:-3]
 350     return clientPath
 351
 352 def currentGitBranch():
 353     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 354
 355 def isValidGitDir(path):
 356     if (os.path.exists(path + "/HEAD")
 357         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 358         return True;
 359     return False
 360
 361 def parseRevision(ref):
 362     return read_pipe("git rev-parse %s" % ref).strip()
 363
 364 def extractLogMessageFromGitCommit(commit):
 365     logMessage = ""
 366
 367     ## fixme: title is first line of commit, not 1st paragraph.
 368     foundTitle = False
 369     for log in read_pipe_lines("git cat-file commit %s" % commit):
 370        if not foundTitle:
 371            if len(log) == 1:
 372                foundTitle = True
 373            continue
 374
 375        logMessage += log
 376     return logMessage
 377
 378 def extractSettingsGitLog(log):
 379     values = {}
 380     for line in log.split("\n"):
 381         line = line.strip()
 382         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 383         if not m:
 384             continue
 385
 386         assignments = m.group(1).split (':')
 387         for a in assignments:
 388             vals = a.split ('=')
 389             key = vals[0].strip()
 390             val = ('='.join (vals[1:])).strip()
 391             if val.endswith ('\"') and val.startswith('"'):
 392                 val = val[1:-1]
 393
 394             values[key] = val
 395
 396     paths = values.get("depot-paths")
 397     if not paths:
 398         paths = values.get("depot-path")
 399     if paths:
 400         values['depot-paths'] = paths.split(',')
 401     return values
 402
 403 def gitBranchExists(branch):
 404     proc = subprocess.Popen(["git", "rev-parse", branch],
 405                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 406     return proc.wait() == 0;
 407
 408 _gitConfig = {}
 409 def gitConfig(key, args = None): # set args to "--bool", for instance
 410     if not _gitConfig.has_key(key):
 411         argsFilter = ""
 412         if args != None:
 413             argsFilter = "%s " % args
 414         cmd = "git config %s%s" % (argsFilter, key)
 415         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 416     return _gitConfig[key]
 417
 418 def gitConfigList(key):
 419     if not _gitConfig.has_key(key):
 420         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 421     return _gitConfig[key]
 422
 423 def p4BranchesInGit(branchesAreInRemotes = True):
 424     branches = {}
 425
 426     cmdline = "git rev-parse --symbolic "
 427     if branchesAreInRemotes:
 428         cmdline += " --remotes"
 429     else:
 430         cmdline += " --branches"
 431
 432     for line in read_pipe_lines(cmdline):
 433         line = line.strip()
 434
 435         ## only import to p4/
 436         if not line.startswith('p4/') or line == "p4/HEAD":
 437             continue
 438         branch = line
 439
 440         # strip off p4
 441         branch = re.sub ("^p4/", "", line)
 442
 443         branches[branch] = parseRevision(line)
 444     return branches
 445
 446 def findUpstreamBranchPoint(head = "HEAD"):
 447     branches = p4BranchesInGit()
 448     # map from depot-path to branch name
 449     branchByDepotPath = {}
 450     for branch in branches.keys():
 451         tip = branches[branch]
 452         log = extractLogMessageFromGitCommit(tip)
 453         settings = extractSettingsGitLog(log)
 454         if settings.has_key("depot-paths"):
 455             paths = ",".join(settings["depot-paths"])
 456             branchByDepotPath[paths] = "remotes/p4/" + branch
 457
 458     settings = None
 459     parent = 0
 460     while parent < 65535:
 461         commit = head + "~%s" % parent
 462         log = extractLogMessageFromGitCommit(commit)
 463         settings = extractSettingsGitLog(log)
 464         if settings.has_key("depot-paths"):
 465             paths = ",".join(settings["depot-paths"])
 466             if branchByDepotPath.has_key(paths):
 467                 return [branchByDepotPath[paths], settings]
 468
 469         parent = parent + 1
 470
 471     return ["", settings]
 472
 473 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 474     if not silent:
 475         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 476                % localRefPrefix)
 477
 478     originPrefix = "origin/p4/"
 479
 480     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 481         line = line.strip()
 482         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 483             continue
 484
 485         headName = line[len(originPrefix):]
 486         remoteHead = localRefPrefix + headName
 487         originHead = line
 488
 489         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 490         if (not original.has_key('depot-paths')
 491             or not original.has_key('change')):
 492             continue
 493
 494         update = False
 495         if not gitBranchExists(remoteHead):
 496             if verbose:
 497                 print "creating %s" % remoteHead
 498             update = True
 499         else:
 500             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 501             if settings.has_key('change') > 0:
 502                 if settings['depot-paths'] == original['depot-paths']:
 503                     originP4Change = int(original['change'])
 504                     p4Change = int(settings['change'])
 505                     if originP4Change > p4Change:
 506                         print ("%s (%s) is newer than %s (%s). "
 507                                "Updating p4 branch from origin."
 508                                % (originHead, originP4Change,
 509                                   remoteHead, p4Change))
 510                         update = True
 511                 else:
 512                     print ("Ignoring: %s was imported from %s while "
 513                            "%s was imported from %s"
 514                            % (originHead, ','.join(original['depot-paths']),
 515                               remoteHead, ','.join(settings['depot-paths'])))
 516
 517         if update:
 518             system("git update-ref %s %s" % (remoteHead, originHead))
 519
 520 def originP4BranchesExist():
 521         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 522
 523 def p4ChangesForPaths(depotPaths, changeRange):
 524     assert depotPaths
 525     cmd = ['changes']
 526     for p in depotPaths:
 527         cmd += ["%s...%s" % (p, changeRange)]
 528     output = p4_read_pipe_lines(cmd)
 529
 530     changes = {}
 531     for line in output:
 532         changeNum = int(line.split(" ")[1])
 533         changes[changeNum] = True
 534
 535     changelist = changes.keys()
 536     changelist.sort()
 537     return changelist
 538
 539 def p4PathStartsWith(path, prefix):
 540     # This method tries to remedy a potential mixed-case issue:
 541     #
 542     # If UserA adds  //depot/DirA/file1
 543     # and UserB adds //depot/dira/file2
 544     #
 545     # we may or may not have a problem. If you have core.ignorecase=true,
 546     # we treat DirA and dira as the same directory
 547     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 548     if ignorecase:
 549         return path.lower().startswith(prefix.lower())
 550     return path.startswith(prefix)
 551
 552 class Command:
 553     def __init__(self):
 554         self.usage = "usage: %prog [options]"
 555         self.needsGit = True
 556
 557 class P4UserMap:
 558     def __init__(self):
 559         self.userMapFromPerforceServer = False
 560
 561     def getUserCacheFilename(self):
 562         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 563         return home + "/.gitp4-usercache.txt"
 564
 565     def getUserMapFromPerforceServer(self):
 566         if self.userMapFromPerforceServer:
 567             return
 568         self.users = {}
 569         self.emails = {}
 570
 571         for output in p4CmdList("users"):
 572             if not output.has_key("User"):
 573                 continue
 574             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 575             self.emails[output["Email"]] = output["User"]
 576
 577
 578         s = ''
 579         for (key, val) in self.users.items():
 580             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 581
 582         open(self.getUserCacheFilename(), "wb").write(s)
 583         self.userMapFromPerforceServer = True
 584
 585     def loadUserMapFromCache(self):
 586         self.users = {}
 587         self.userMapFromPerforceServer = False
 588         try:
 589             cache = open(self.getUserCacheFilename(), "rb")
 590             lines = cache.readlines()
 591             cache.close()
 592             for line in lines:
 593                 entry = line.strip().split("\t")
 594                 self.users[entry[0]] = entry[1]
 595         except IOError:
 596             self.getUserMapFromPerforceServer()
 597
 598 class P4Debug(Command):
 599     def __init__(self):
 600         Command.__init__(self)
 601         self.options = [
 602             optparse.make_option("--verbose", dest="verbose", action="store_true",
 603                                  default=False),
 604             ]
 605         self.description = "A tool to debug the output of p4 -G."
 606         self.needsGit = False
 607         self.verbose = False
 608
 609     def run(self, args):
 610         j = 0
 611         for output in p4CmdList(args):
 612             print 'Element: %d' % j
 613             j += 1
 614             print output
 615         return True
 616
 617 class P4RollBack(Command):
 618     def __init__(self):
 619         Command.__init__(self)
 620         self.options = [
 621             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 622             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 623         ]
 624         self.description = "A tool to debug the multi-branch import. Don't use :)"
 625         self.verbose = False
 626         self.rollbackLocalBranches = False
 627
 628     def run(self, args):
 629         if len(args) != 1:
 630             return False
 631         maxChange = int(args[0])
 632
 633         if "p4ExitCode" in p4Cmd("changes -m 1"):
 634             die("Problems executing p4");
 635
 636         if self.rollbackLocalBranches:
 637             refPrefix = "refs/heads/"
 638             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 639         else:
 640             refPrefix = "refs/remotes/"
 641             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 642
 643         for line in lines:
 644             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 645                 line = line.strip()
 646                 ref = refPrefix + line
 647                 log = extractLogMessageFromGitCommit(ref)
 648                 settings = extractSettingsGitLog(log)
 649
 650                 depotPaths = settings['depot-paths']
 651                 change = settings['change']
 652
 653                 changed = False
 654
 655                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 656                                                            for p in depotPaths]))) == 0:
 657                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 658                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 659                     continue
 660
 661                 while change and int(change) > maxChange:
 662                     changed = True
 663                     if self.verbose:
 664                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 665                     system("git update-ref %s \"%s^\"" % (ref, ref))
 666                     log = extractLogMessageFromGitCommit(ref)
 667                     settings =  extractSettingsGitLog(log)
 668
 669
 670                     depotPaths = settings['depot-paths']
 671                     change = settings['change']
 672
 673                 if changed:
 674                     print "%s rewound to %s" % (ref, change)
 675
 676         return True
 677
 678 class P4Submit(Command, P4UserMap):
 679     def __init__(self):
 680         Command.__init__(self)
 681         P4UserMap.__init__(self)
 682         self.options = [
 683                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 684                 optparse.make_option("--origin", dest="origin"),
 685                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 686                 # preserve the user, requires relevant p4 permissions
 687                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 688         ]
 689         self.description = "Submit changes from git to the perforce depot."
 690         self.usage += " [name of git branch to submit into perforce depot]"
 691         self.interactive = True
 692         self.origin = ""
 693         self.detectRenames = False
 694         self.verbose = False
 695         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 696         self.isWindows = (platform.system() == "Windows")
 697         self.myP4UserId = None
 698
 699     def check(self):
 700         if len(p4CmdList("opened ...")) > 0:
 701             die("You have files opened with perforce! Close them before starting the sync.")
 702
 703     # replaces everything between 'Description:' and the next P4 submit template field with the
 704     # commit message
 705     def prepareLogMessage(self, template, message):
 706         result = ""
 707
 708         inDescriptionSection = False
 709
 710         for line in template.split("\n"):
 711             if line.startswith("#"):
 712                 result += line + "\n"
 713                 continue
 714
 715             if inDescriptionSection:
 716                 if line.startswith("Files:") or line.startswith("Jobs:"):
 717                     inDescriptionSection = False
 718                 else:
 719                     continue
 720             else:
 721                 if line.startswith("Description:"):
 722                     inDescriptionSection = True
 723                     line += "\n"
 724                     for messageLine in message.split("\n"):
 725                         line += "\t" + messageLine + "\n"
 726
 727             result += line + "\n"
 728
 729         return result
 730
 731     def p4UserForCommit(self,id):
 732         # Return the tuple (perforce user,git email) for a given git commit id
 733         self.getUserMapFromPerforceServer()
 734         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 735         gitEmail = gitEmail.strip()
 736         if not self.emails.has_key(gitEmail):
 737             return (None,gitEmail)
 738         else:
 739             return (self.emails[gitEmail],gitEmail)
 740
 741     def checkValidP4Users(self,commits):
 742         # check if any git authors cannot be mapped to p4 users
 743         for id in commits:
 744             (user,email) = self.p4UserForCommit(id)
 745             if not user:
 746                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 747                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 748                     print "%s" % msg
 749                 else:
 750                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 751
 752     def lastP4Changelist(self):
 753         # Get back the last changelist number submitted in this client spec. This
 754         # then gets used to patch up the username in the change. If the same
 755         # client spec is being used by multiple processes then this might go
 756         # wrong.
 757         results = p4CmdList("client -o")        # find the current client
 758         client = None
 759         for r in results:
 760             if r.has_key('Client'):
 761                 client = r['Client']
 762                 break
 763         if not client:
 764             die("could not get client spec")
 765         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 766         for r in results:
 767             if r.has_key('change'):
 768                 return r['change']
 769         die("Could not get changelist number for last submit - cannot patch up user details")
 770
 771     def modifyChangelistUser(self, changelist, newUser):
 772         # fixup the user field of a changelist after it has been submitted.
 773         changes = p4CmdList("change -o %s" % changelist)
 774         if len(changes) != 1:
 775             die("Bad output from p4 change modifying %s to user %s" %
 776                 (changelist, newUser))
 777
 778         c = changes[0]
 779         if c['User'] == newUser: return   # nothing to do
 780         c['User'] = newUser
 781         input = marshal.dumps(c)
 782
 783         result = p4CmdList("change -f -i", stdin=input)
 784         for r in result:
 785             if r.has_key('code'):
 786                 if r['code'] == 'error':
 787                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 788             if r.has_key('data'):
 789                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 790                 return
 791         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 792
 793     def canChangeChangelists(self):
 794         # check to see if we have p4 admin or super-user permissions, either of
 795         # which are required to modify changelists.
 796         results = p4CmdList("protects %s" % self.depotPath)
 797         for r in results:
 798             if r.has_key('perm'):
 799                 if r['perm'] == 'admin':
 800                     return 1
 801                 if r['perm'] == 'super':
 802                     return 1
 803         return 0
 804
 805     def p4UserId(self):
 806         if self.myP4UserId:
 807             return self.myP4UserId
 808
 809         results = p4CmdList("user -o")
 810         for r in results:
 811             if r.has_key('User'):
 812                 self.myP4UserId = r['User']
 813                 return r['User']
 814         die("Could not find your p4 user id")
 815
 816     def p4UserIsMe(self, p4User):
 817         # return True if the given p4 user is actually me
 818         me = self.p4UserId()
 819         if not p4User or p4User != me:
 820             return False
 821         else:
 822             return True
 823
 824     def prepareSubmitTemplate(self):
 825         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 826         template = ""
 827         inFilesSection = False
 828         for line in p4_read_pipe_lines(['change', '-o']):
 829             if line.endswith("\r\n"):
 830                 line = line[:-2] + "\n"
 831             if inFilesSection:
 832                 if line.startswith("\t"):
 833                     # path starts and ends with a tab
 834                     path = line[1:]
 835                     lastTab = path.rfind("\t")
 836                     if lastTab != -1:
 837                         path = path[:lastTab]
 838                         if not p4PathStartsWith(path, self.depotPath):
 839                             continue
 840                 else:
 841                     inFilesSection = False
 842             else:
 843                 if line.startswith("Files:"):
 844                     inFilesSection = True
 845
 846             template += line
 847
 848         return template
 849
 850     def applyCommit(self, id):
 851         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 852
 853         (p4User, gitEmail) = self.p4UserForCommit(id)
 854
 855         if not self.detectRenames:
 856             # If not explicitly set check the config variable
 857             self.detectRenames = gitConfig("git-p4.detectRenames")
 858
 859         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 860             diffOpts = ""
 861         elif self.detectRenames.lower() == "true":
 862             diffOpts = "-M"
 863         else:
 864             diffOpts = "-M%s" % self.detectRenames
 865
 866         detectCopies = gitConfig("git-p4.detectCopies")
 867         if detectCopies.lower() == "true":
 868             diffOpts += " -C"
 869         elif detectCopies != "" and detectCopies.lower() != "false":
 870             diffOpts += " -C%s" % detectCopies
 871
 872         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 873             diffOpts += " --find-copies-harder"
 874
 875         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 876         filesToAdd = set()
 877         filesToDelete = set()
 878         editedFiles = set()
 879         filesToChangeExecBit = {}
 880         for line in diff:
 881             diff = parseDiffTreeEntry(line)
 882             modifier = diff['status']
 883             path = diff['src']
 884             if modifier == "M":
 885                 p4_edit(path)
 886                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 887                     filesToChangeExecBit[path] = diff['dst_mode']
 888                 editedFiles.add(path)
 889             elif modifier == "A":
 890                 filesToAdd.add(path)
 891                 filesToChangeExecBit[path] = diff['dst_mode']
 892                 if path in filesToDelete:
 893                     filesToDelete.remove(path)
 894             elif modifier == "D":
 895                 filesToDelete.add(path)
 896                 if path in filesToAdd:
 897                     filesToAdd.remove(path)
 898             elif modifier == "C":
 899                 src, dest = diff['src'], diff['dst']
 900                 p4_integrate(src, dest)
 901                 if diff['src_sha1'] != diff['dst_sha1']:
 902                     p4_edit(dest)
 903                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 904                     p4_edit(dest)
 905                     filesToChangeExecBit[dest] = diff['dst_mode']
 906                 os.unlink(dest)
 907                 editedFiles.add(dest)
 908             elif modifier == "R":
 909                 src, dest = diff['src'], diff['dst']
 910                 p4_integrate(src, dest)
 911                 if diff['src_sha1'] != diff['dst_sha1']:
 912                     p4_edit(dest)
 913                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 914                     p4_edit(dest)
 915                     filesToChangeExecBit[dest] = diff['dst_mode']
 916                 os.unlink(dest)
 917                 editedFiles.add(dest)
 918                 filesToDelete.add(src)
 919             else:
 920                 die("unknown modifier %s for %s" % (modifier, path))
 921
 922         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 923         patchcmd = diffcmd + " | git apply "
 924         tryPatchCmd = patchcmd + "--check -"
 925         applyPatchCmd = patchcmd + "--check --apply -"
 926
 927         if os.system(tryPatchCmd) != 0:
 928             print "Unfortunately applying the change failed!"
 929             print "What do you want to do?"
 930             response = "x"
 931             while response != "s" and response != "a" and response != "w":
 932                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 933                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 934             if response == "s":
 935                 print "Skipping! Good luck with the next patches..."
 936                 for f in editedFiles:
 937                     p4_revert(f)
 938                 for f in filesToAdd:
 939                     os.remove(f)
 940                 return
 941             elif response == "a":
 942                 os.system(applyPatchCmd)
 943                 if len(filesToAdd) > 0:
 944                     print "You may also want to call p4 add on the following files:"
 945                     print " ".join(filesToAdd)
 946                 if len(filesToDelete):
 947                     print "The following files should be scheduled for deletion with p4 delete:"
 948                     print " ".join(filesToDelete)
 949                 die("Please resolve and submit the conflict manually and "
 950                     + "continue afterwards with git-p4 submit --continue")
 951             elif response == "w":
 952                 system(diffcmd + " > patch.txt")
 953                 print "Patch saved to patch.txt in %s !" % self.clientPath
 954                 die("Please resolve and submit the conflict manually and "
 955                     "continue afterwards with git-p4 submit --continue")
 956
 957         system(applyPatchCmd)
 958
 959         for f in filesToAdd:
 960             p4_add(f)
 961         for f in filesToDelete:
 962             p4_revert(f)
 963             p4_delete(f)
 964
 965         # Set/clear executable bits
 966         for f in filesToChangeExecBit.keys():
 967             mode = filesToChangeExecBit[f]
 968             setP4ExecBit(f, mode)
 969
 970         logMessage = extractLogMessageFromGitCommit(id)
 971         logMessage = logMessage.strip()
 972
 973         template = self.prepareSubmitTemplate()
 974
 975         if self.interactive:
 976             submitTemplate = self.prepareLogMessage(template, logMessage)
 977
 978             if self.preserveUser:
 979                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
 980
 981             if os.environ.has_key("P4DIFF"):
 982                 del(os.environ["P4DIFF"])
 983             diff = ""
 984             for editedFile in editedFiles:
 985                 diff += p4_read_pipe(['diff', '-du', editedFile])
 986
 987             newdiff = ""
 988             for newFile in filesToAdd:
 989                 newdiff += "==== new file ====\n"
 990                 newdiff += "--- /dev/null\n"
 991                 newdiff += "+++ %s\n" % newFile
 992                 f = open(newFile, "r")
 993                 for line in f.readlines():
 994                     newdiff += "+" + line
 995                 f.close()
 996
 997             if self.checkAuthorship and not self.p4UserIsMe(p4User):
 998                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
 999                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1000                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1001
1002             separatorLine = "######## everything below this line is just the diff #######\n"
1003
1004             [handle, fileName] = tempfile.mkstemp()
1005             tmpFile = os.fdopen(handle, "w+")
1006             if self.isWindows:
1007                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1008                 separatorLine = separatorLine.replace("\n", "\r\n")
1009                 newdiff = newdiff.replace("\n", "\r\n")
1010             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1011             tmpFile.close()
1012             mtime = os.stat(fileName).st_mtime
1013             if os.environ.has_key("P4EDITOR"):
1014                 editor = os.environ.get("P4EDITOR")
1015             else:
1016                 editor = read_pipe("git var GIT_EDITOR").strip()
1017             system(editor + " " + fileName)
1018
1019             if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1020                 checkModTime = False
1021             else:
1022                 checkModTime = True
1023
1024             response = "y"
1025             if checkModTime and (os.stat(fileName).st_mtime <= mtime):
1026                 response = "x"
1027                 while response != "y" and response != "n":
1028                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1029
1030             if response == "y":
1031                 tmpFile = open(fileName, "rb")
1032                 message = tmpFile.read()
1033                 tmpFile.close()
1034                 submitTemplate = message[:message.index(separatorLine)]
1035                 if self.isWindows:
1036                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1037                 p4_write_pipe(['submit', '-i'], submitTemplate)
1038
1039                 if self.preserveUser:
1040                     if p4User:
1041                         # Get last changelist number. Cannot easily get it from
1042                         # the submit command output as the output is unmarshalled.
1043                         changelist = self.lastP4Changelist()
1044                         self.modifyChangelistUser(changelist, p4User)
1045
1046             else:
1047                 for f in editedFiles:
1048                     p4_revert(f)
1049                 for f in filesToAdd:
1050                     p4_revert(f)
1051                     os.remove(f)
1052
1053             os.remove(fileName)
1054         else:
1055             fileName = "submit.txt"
1056             file = open(fileName, "w+")
1057             file.write(self.prepareLogMessage(template, logMessage))
1058             file.close()
1059             print ("Perforce submit template written as %s. "
1060                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1061                    % (fileName, fileName))
1062
1063     def run(self, args):
1064         if len(args) == 0:
1065             self.master = currentGitBranch()
1066             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1067                 die("Detecting current git branch failed!")
1068         elif len(args) == 1:
1069             self.master = args[0]
1070         else:
1071             return False
1072
1073         allowSubmit = gitConfig("git-p4.allowSubmit")
1074         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1075             die("%s is not in git-p4.allowSubmit" % self.master)
1076
1077         [upstream, settings] = findUpstreamBranchPoint()
1078         self.depotPath = settings['depot-paths'][0]
1079         if len(self.origin) == 0:
1080             self.origin = upstream
1081
1082         if self.preserveUser:
1083             if not self.canChangeChangelists():
1084                 die("Cannot preserve user names without p4 super-user or admin permissions")
1085
1086         if self.verbose:
1087             print "Origin branch is " + self.origin
1088
1089         if len(self.depotPath) == 0:
1090             print "Internal error: cannot locate perforce depot path from existing branches"
1091             sys.exit(128)
1092
1093         self.clientPath = p4Where(self.depotPath)
1094
1095         if len(self.clientPath) == 0:
1096             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1097             sys.exit(128)
1098
1099         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1100         self.oldWorkingDirectory = os.getcwd()
1101
1102         chdir(self.clientPath)
1103         print "Synchronizing p4 checkout..."
1104         p4_sync("...")
1105         self.check()
1106
1107         commits = []
1108         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1109             commits.append(line.strip())
1110         commits.reverse()
1111
1112         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1113             self.checkAuthorship = False
1114         else:
1115             self.checkAuthorship = True
1116
1117         if self.preserveUser:
1118             self.checkValidP4Users(commits)
1119
1120         while len(commits) > 0:
1121             commit = commits[0]
1122             commits = commits[1:]
1123             self.applyCommit(commit)
1124             if not self.interactive:
1125                 break
1126
1127         if len(commits) == 0:
1128             print "All changes applied!"
1129             chdir(self.oldWorkingDirectory)
1130
1131             sync = P4Sync()
1132             sync.run([])
1133
1134             rebase = P4Rebase()
1135             rebase.rebase()
1136
1137         return True
1138
1139 class P4Sync(Command, P4UserMap):
1140     delete_actions = ( "delete", "move/delete", "purge" )
1141
1142     def __init__(self):
1143         Command.__init__(self)
1144         P4UserMap.__init__(self)
1145         self.options = [
1146                 optparse.make_option("--branch", dest="branch"),
1147                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1148                 optparse.make_option("--changesfile", dest="changesFile"),
1149                 optparse.make_option("--silent", dest="silent", action="store_true"),
1150                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1151                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1152                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1153                                      help="Import into refs/heads/ , not refs/remotes"),
1154                 optparse.make_option("--max-changes", dest="maxChanges"),
1155                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1156                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1157                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1158                                      help="Only sync files that are included in the Perforce Client Spec")
1159         ]
1160         self.description = """Imports from Perforce into a git repository.\n
1161     example:
1162     //depot/my/project/ -- to import the current head
1163     //depot/my/project/@all -- to import everything
1164     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1165
1166     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1167
1168         self.usage += " //depot/path[@revRange]"
1169         self.silent = False
1170         self.createdBranches = set()
1171         self.committedChanges = set()
1172         self.branch = ""
1173         self.detectBranches = False
1174         self.detectLabels = False
1175         self.changesFile = ""
1176         self.syncWithOrigin = True
1177         self.verbose = False
1178         self.importIntoRemotes = True
1179         self.maxChanges = ""
1180         self.isWindows = (platform.system() == "Windows")
1181         self.keepRepoPath = False
1182         self.depotPaths = None
1183         self.p4BranchesInGit = []
1184         self.cloneExclude = []
1185         self.useClientSpec = False
1186         self.clientSpecDirs = []
1187
1188         if gitConfig("git-p4.syncFromOrigin") == "false":
1189             self.syncWithOrigin = False
1190
1191     #
1192     # P4 wildcards are not allowed in filenames.  P4 complains
1193     # if you simply add them, but you can force it with "-f", in
1194     # which case it translates them into %xx encoding internally.
1195     # Search for and fix just these four characters.  Do % last so
1196     # that fixing it does not inadvertently create new %-escapes.
1197     #
1198     def wildcard_decode(self, path):
1199         # Cannot have * in a filename in windows; untested as to
1200         # what p4 would do in such a case.
1201         if not self.isWindows:
1202             path = path.replace("%2A", "*")
1203         path = path.replace("%23", "#") \
1204                    .replace("%40", "@") \
1205                    .replace("%25", "%")
1206         return path
1207
1208     def extractFilesFromCommit(self, commit):
1209         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1210                              for path in self.cloneExclude]
1211         files = []
1212         fnum = 0
1213         while commit.has_key("depotFile%s" % fnum):
1214             path =  commit["depotFile%s" % fnum]
1215
1216             if [p for p in self.cloneExclude
1217                 if p4PathStartsWith(path, p)]:
1218                 found = False
1219             else:
1220                 found = [p for p in self.depotPaths
1221                          if p4PathStartsWith(path, p)]
1222             if not found:
1223                 fnum = fnum + 1
1224                 continue
1225
1226             file = {}
1227             file["path"] = path
1228             file["rev"] = commit["rev%s" % fnum]
1229             file["action"] = commit["action%s" % fnum]
1230             file["type"] = commit["type%s" % fnum]
1231             files.append(file)
1232             fnum = fnum + 1
1233         return files
1234
1235     def stripRepoPath(self, path, prefixes):
1236         if self.useClientSpec:
1237
1238             # if using the client spec, we use the output directory
1239             # specified in the client.  For example, a view
1240             #   //depot/foo/branch/... //client/branch/foo/...
1241             # will end up putting all foo/branch files into
1242             #  branch/foo/
1243             for val in self.clientSpecDirs:
1244                 if path.startswith(val[0]):
1245                     # replace the depot path with the client path
1246                     path = path.replace(val[0], val[1][1])
1247                     # now strip out the client (//client/...)
1248                     path = re.sub("^(//[^/]+/)", '', path)
1249                     # the rest is all path
1250                     return path
1251
1252         if self.keepRepoPath:
1253             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1254
1255         for p in prefixes:
1256             if p4PathStartsWith(path, p):
1257                 path = path[len(p):]
1258
1259         return path
1260
1261     def splitFilesIntoBranches(self, commit):
1262         branches = {}
1263         fnum = 0
1264         while commit.has_key("depotFile%s" % fnum):
1265             path =  commit["depotFile%s" % fnum]
1266             found = [p for p in self.depotPaths
1267                      if p4PathStartsWith(path, p)]
1268             if not found:
1269                 fnum = fnum + 1
1270                 continue
1271
1272             file = {}
1273             file["path"] = path
1274             file["rev"] = commit["rev%s" % fnum]
1275             file["action"] = commit["action%s" % fnum]
1276             file["type"] = commit["type%s" % fnum]
1277             fnum = fnum + 1
1278
1279             relPath = self.stripRepoPath(path, self.depotPaths)
1280
1281             for branch in self.knownBranches.keys():
1282
1283                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1284                 if relPath.startswith(branch + "/"):
1285                     if branch not in branches:
1286                         branches[branch] = []
1287                     branches[branch].append(file)
1288                     break
1289
1290         return branches
1291
1292     # output one file from the P4 stream
1293     # - helper for streamP4Files
1294
1295     def streamOneP4File(self, file, contents):
1296         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1297         relPath = self.wildcard_decode(relPath)
1298         if verbose:
1299             sys.stderr.write("%s\n" % relPath)
1300
1301         (type_base, type_mods) = split_p4_type(file["type"])
1302
1303         git_mode = "100644"
1304         if "x" in type_mods:
1305             git_mode = "100755"
1306         if type_base == "symlink":
1307             git_mode = "120000"
1308             # p4 print on a symlink contains "target\n"; remove the newline
1309             data = ''.join(contents)
1310             contents = [data[:-1]]
1311
1312         if type_base == "utf16":
1313             # p4 delivers different text in the python output to -G
1314             # than it does when using "print -o", or normal p4 client
1315             # operations.  utf16 is converted to ascii or utf8, perhaps.
1316             # But ascii text saved as -t utf16 is completely mangled.
1317             # Invoke print -o to get the real contents.
1318             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1319             contents = [ text ]
1320
1321         if type_base == "apple":
1322             # Apple filetype files will be streamed as a concatenation of
1323             # its appledouble header and the contents.  This is useless
1324             # on both macs and non-macs.  If using "print -q -o xx", it
1325             # will create "xx" with the data, and "%xx" with the header.
1326             # This is also not very useful.
1327             #
1328             # Ideally, someday, this script can learn how to generate
1329             # appledouble files directly and import those to git, but
1330             # non-mac machines can never find a use for apple filetype.
1331             print "\nIgnoring apple filetype file %s" % file['depotFile']
1332             return
1333
1334         # Perhaps windows wants unicode, utf16 newlines translated too;
1335         # but this is not doing it.
1336         if self.isWindows and type_base == "text":
1337             mangled = []
1338             for data in contents:
1339                 data = data.replace("\r\n", "\n")
1340                 mangled.append(data)
1341             contents = mangled
1342
1343         # Note that we do not try to de-mangle keywords on utf16 files,
1344         # even though in theory somebody may want that.
1345         if type_base in ("text", "unicode", "binary"):
1346             if "ko" in type_mods:
1347                 text = ''.join(contents)
1348                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1349                 contents = [ text ]
1350             elif "k" in type_mods:
1351                 text = ''.join(contents)
1352                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1353                 contents = [ text ]
1354
1355         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1356
1357         # total length...
1358         length = 0
1359         for d in contents:
1360             length = length + len(d)
1361
1362         self.gitStream.write("data %d\n" % length)
1363         for d in contents:
1364             self.gitStream.write(d)
1365         self.gitStream.write("\n")
1366
1367     def streamOneP4Deletion(self, file):
1368         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1369         if verbose:
1370             sys.stderr.write("delete %s\n" % relPath)
1371         self.gitStream.write("D %s\n" % relPath)
1372
1373     # handle another chunk of streaming data
1374     def streamP4FilesCb(self, marshalled):
1375
1376         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1377             # start of a new file - output the old one first
1378             self.streamOneP4File(self.stream_file, self.stream_contents)
1379             self.stream_file = {}
1380             self.stream_contents = []
1381             self.stream_have_file_info = False
1382
1383         # pick up the new file information... for the
1384         # 'data' field we need to append to our array
1385         for k in marshalled.keys():
1386             if k == 'data':
1387                 self.stream_contents.append(marshalled['data'])
1388             else:
1389                 self.stream_file[k] = marshalled[k]
1390
1391         self.stream_have_file_info = True
1392
1393     # Stream directly from "p4 files" into "git fast-import"
1394     def streamP4Files(self, files):
1395         filesForCommit = []
1396         filesToRead = []
1397         filesToDelete = []
1398
1399         for f in files:
1400             includeFile = True
1401             for val in self.clientSpecDirs:
1402                 if f['path'].startswith(val[0]):
1403                     if val[1][0] <= 0:
1404                         includeFile = False
1405                     break
1406
1407             if includeFile:
1408                 filesForCommit.append(f)
1409                 if f['action'] in self.delete_actions:
1410                     filesToDelete.append(f)
1411                 else:
1412                     filesToRead.append(f)
1413
1414         # deleted files...
1415         for f in filesToDelete:
1416             self.streamOneP4Deletion(f)
1417
1418         if len(filesToRead) > 0:
1419             self.stream_file = {}
1420             self.stream_contents = []
1421             self.stream_have_file_info = False
1422
1423             # curry self argument
1424             def streamP4FilesCbSelf(entry):
1425                 self.streamP4FilesCb(entry)
1426
1427             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1428
1429             p4CmdList(["-x", "-", "print"],
1430                       stdin=fileArgs,
1431                       cb=streamP4FilesCbSelf)
1432
1433             # do the last chunk
1434             if self.stream_file.has_key('depotFile'):
1435                 self.streamOneP4File(self.stream_file, self.stream_contents)
1436
1437     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1438         epoch = details["time"]
1439         author = details["user"]
1440         self.branchPrefixes = branchPrefixes
1441
1442         if self.verbose:
1443             print "commit into %s" % branch
1444
1445         # start with reading files; if that fails, we should not
1446         # create a commit.
1447         new_files = []
1448         for f in files:
1449             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1450                 new_files.append (f)
1451             else:
1452                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1453
1454         self.gitStream.write("commit %s\n" % branch)
1455 #        gitStream.write("mark :%s\n" % details["change"])
1456         self.committedChanges.add(int(details["change"]))
1457         committer = ""
1458         if author not in self.users:
1459             self.getUserMapFromPerforceServer()
1460         if author in self.users:
1461             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1462         else:
1463             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1464
1465         self.gitStream.write("committer %s\n" % committer)
1466
1467         self.gitStream.write("data <<EOT\n")
1468         self.gitStream.write(details["desc"])
1469         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1470                              % (','.join (branchPrefixes), details["change"]))
1471         if len(details['options']) > 0:
1472             self.gitStream.write(": options = %s" % details['options'])
1473         self.gitStream.write("]\nEOT\n\n")
1474
1475         if len(parent) > 0:
1476             if self.verbose:
1477                 print "parent %s" % parent
1478             self.gitStream.write("from %s\n" % parent)
1479
1480         self.streamP4Files(new_files)
1481         self.gitStream.write("\n")
1482
1483         change = int(details["change"])
1484
1485         if self.labels.has_key(change):
1486             label = self.labels[change]
1487             labelDetails = label[0]
1488             labelRevisions = label[1]
1489             if self.verbose:
1490                 print "Change %s is labelled %s" % (change, labelDetails)
1491
1492             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1493                                                     for p in branchPrefixes])
1494
1495             if len(files) == len(labelRevisions):
1496
1497                 cleanedFiles = {}
1498                 for info in files:
1499                     if info["action"] in self.delete_actions:
1500                         continue
1501                     cleanedFiles[info["depotFile"]] = info["rev"]
1502
1503                 if cleanedFiles == labelRevisions:
1504                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1505                     self.gitStream.write("from %s\n" % branch)
1506
1507                     owner = labelDetails["Owner"]
1508                     tagger = ""
1509                     if author in self.users:
1510                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1511                     else:
1512                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1513                     self.gitStream.write("tagger %s\n" % tagger)
1514                     self.gitStream.write("data <<EOT\n")
1515                     self.gitStream.write(labelDetails["Description"])
1516                     self.gitStream.write("EOT\n\n")
1517
1518                 else:
1519                     if not self.silent:
1520                         print ("Tag %s does not match with change %s: files do not match."
1521                                % (labelDetails["label"], change))
1522
1523             else:
1524                 if not self.silent:
1525                     print ("Tag %s does not match with change %s: file count is different."
1526                            % (labelDetails["label"], change))
1527
1528     def getLabels(self):
1529         self.labels = {}
1530
1531         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1532         if len(l) > 0 and not self.silent:
1533             print "Finding files belonging to labels in %s" % `self.depotPaths`
1534
1535         for output in l:
1536             label = output["label"]
1537             revisions = {}
1538             newestChange = 0
1539             if self.verbose:
1540                 print "Querying files for label %s" % label
1541             for file in p4CmdList(["files"] +
1542                                       ["%s...@%s" % (p, label)
1543                                           for p in self.depotPaths]):
1544                 revisions[file["depotFile"]] = file["rev"]
1545                 change = int(file["change"])
1546                 if change > newestChange:
1547                     newestChange = change
1548
1549             self.labels[newestChange] = [output, revisions]
1550
1551         if self.verbose:
1552             print "Label changes: %s" % self.labels.keys()
1553
1554     def guessProjectName(self):
1555         for p in self.depotPaths:
1556             if p.endswith("/"):
1557                 p = p[:-1]
1558             p = p[p.strip().rfind("/") + 1:]
1559             if not p.endswith("/"):
1560                p += "/"
1561             return p
1562
1563     def getBranchMapping(self):
1564         lostAndFoundBranches = set()
1565
1566         user = gitConfig("git-p4.branchUser")
1567         if len(user) > 0:
1568             command = "branches -u %s" % user
1569         else:
1570             command = "branches"
1571
1572         for info in p4CmdList(command):
1573             details = p4Cmd("branch -o %s" % info["branch"])
1574             viewIdx = 0
1575             while details.has_key("View%s" % viewIdx):
1576                 paths = details["View%s" % viewIdx].split(" ")
1577                 viewIdx = viewIdx + 1
1578                 # require standard //depot/foo/... //depot/bar/... mapping
1579                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1580                     continue
1581                 source = paths[0]
1582                 destination = paths[1]
1583                 ## HACK
1584                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1585                     source = source[len(self.depotPaths[0]):-4]
1586                     destination = destination[len(self.depotPaths[0]):-4]
1587
1588                     if destination in self.knownBranches:
1589                         if not self.silent:
1590                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1591                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1592                         continue
1593
1594                     self.knownBranches[destination] = source
1595
1596                     lostAndFoundBranches.discard(destination)
1597
1598                     if source not in self.knownBranches:
1599                         lostAndFoundBranches.add(source)
1600
1601         # Perforce does not strictly require branches to be defined, so we also
1602         # check git config for a branch list.
1603         #
1604         # Example of branch definition in git config file:
1605         # [git-p4]
1606         #   branchList=main:branchA
1607         #   branchList=main:branchB
1608         #   branchList=branchA:branchC
1609         configBranches = gitConfigList("git-p4.branchList")
1610         for branch in configBranches:
1611             if branch:
1612                 (source, destination) = branch.split(":")
1613                 self.knownBranches[destination] = source
1614
1615                 lostAndFoundBranches.discard(destination)
1616
1617                 if source not in self.knownBranches:
1618                     lostAndFoundBranches.add(source)
1619
1620
1621         for branch in lostAndFoundBranches:
1622             self.knownBranches[branch] = branch
1623
1624     def getBranchMappingFromGitBranches(self):
1625         branches = p4BranchesInGit(self.importIntoRemotes)
1626         for branch in branches.keys():
1627             if branch == "master":
1628                 branch = "main"
1629             else:
1630                 branch = branch[len(self.projectName):]
1631             self.knownBranches[branch] = branch
1632
1633     def listExistingP4GitBranches(self):
1634         # branches holds mapping from name to commit
1635         branches = p4BranchesInGit(self.importIntoRemotes)
1636         self.p4BranchesInGit = branches.keys()
1637         for branch in branches.keys():
1638             self.initialParents[self.refPrefix + branch] = branches[branch]
1639
1640     def updateOptionDict(self, d):
1641         option_keys = {}
1642         if self.keepRepoPath:
1643             option_keys['keepRepoPath'] = 1
1644
1645         d["options"] = ' '.join(sorted(option_keys.keys()))
1646
1647     def readOptions(self, d):
1648         self.keepRepoPath = (d.has_key('options')
1649                              and ('keepRepoPath' in d['options']))
1650
1651     def gitRefForBranch(self, branch):
1652         if branch == "main":
1653             return self.refPrefix + "master"
1654
1655         if len(branch) <= 0:
1656             return branch
1657
1658         return self.refPrefix + self.projectName + branch
1659
1660     def gitCommitByP4Change(self, ref, change):
1661         if self.verbose:
1662             print "looking in ref " + ref + " for change %s using bisect..." % change
1663
1664         earliestCommit = ""
1665         latestCommit = parseRevision(ref)
1666
1667         while True:
1668             if self.verbose:
1669                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1670             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1671             if len(next) == 0:
1672                 if self.verbose:
1673                     print "argh"
1674                 return ""
1675             log = extractLogMessageFromGitCommit(next)
1676             settings = extractSettingsGitLog(log)
1677             currentChange = int(settings['change'])
1678             if self.verbose:
1679                 print "current change %s" % currentChange
1680
1681             if currentChange == change:
1682                 if self.verbose:
1683                     print "found %s" % next
1684                 return next
1685
1686             if currentChange < change:
1687                 earliestCommit = "^%s" % next
1688             else:
1689                 latestCommit = "%s" % next
1690
1691         return ""
1692
1693     def importNewBranch(self, branch, maxChange):
1694         # make fast-import flush all changes to disk and update the refs using the checkpoint
1695         # command so that we can try to find the branch parent in the git history
1696         self.gitStream.write("checkpoint\n\n");
1697         self.gitStream.flush();
1698         branchPrefix = self.depotPaths[0] + branch + "/"
1699         range = "@1,%s" % maxChange
1700         #print "prefix" + branchPrefix
1701         changes = p4ChangesForPaths([branchPrefix], range)
1702         if len(changes) <= 0:
1703             return False
1704         firstChange = changes[0]
1705         #print "first change in branch: %s" % firstChange
1706         sourceBranch = self.knownBranches[branch]
1707         sourceDepotPath = self.depotPaths[0] + sourceBranch
1708         sourceRef = self.gitRefForBranch(sourceBranch)
1709         #print "source " + sourceBranch
1710
1711         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1712         #print "branch parent: %s" % branchParentChange
1713         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1714         if len(gitParent) > 0:
1715             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1716             #print "parent git commit: %s" % gitParent
1717
1718         self.importChanges(changes)
1719         return True
1720
1721     def importChanges(self, changes):
1722         cnt = 1
1723         for change in changes:
1724             description = p4Cmd("describe %s" % change)
1725             self.updateOptionDict(description)
1726
1727             if not self.silent:
1728                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1729                 sys.stdout.flush()
1730             cnt = cnt + 1
1731
1732             try:
1733                 if self.detectBranches:
1734                     branches = self.splitFilesIntoBranches(description)
1735                     for branch in branches.keys():
1736                         ## HACK  --hwn
1737                         branchPrefix = self.depotPaths[0] + branch + "/"
1738
1739                         parent = ""
1740
1741                         filesForCommit = branches[branch]
1742
1743                         if self.verbose:
1744                             print "branch is %s" % branch
1745
1746                         self.updatedBranches.add(branch)
1747
1748                         if branch not in self.createdBranches:
1749                             self.createdBranches.add(branch)
1750                             parent = self.knownBranches[branch]
1751                             if parent == branch:
1752                                 parent = ""
1753                             else:
1754                                 fullBranch = self.projectName + branch
1755                                 if fullBranch not in self.p4BranchesInGit:
1756                                     if not self.silent:
1757                                         print("\n    Importing new branch %s" % fullBranch);
1758                                     if self.importNewBranch(branch, change - 1):
1759                                         parent = ""
1760                                         self.p4BranchesInGit.append(fullBranch)
1761                                     if not self.silent:
1762                                         print("\n    Resuming with change %s" % change);
1763
1764                                 if self.verbose:
1765                                     print "parent determined through known branches: %s" % parent
1766
1767                         branch = self.gitRefForBranch(branch)
1768                         parent = self.gitRefForBranch(parent)
1769
1770                         if self.verbose:
1771                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1772
1773                         if len(parent) == 0 and branch in self.initialParents:
1774                             parent = self.initialParents[branch]
1775                             del self.initialParents[branch]
1776
1777                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1778                 else:
1779                     files = self.extractFilesFromCommit(description)
1780                     self.commit(description, files, self.branch, self.depotPaths,
1781                                 self.initialParent)
1782                     self.initialParent = ""
1783             except IOError:
1784                 print self.gitError.read()
1785                 sys.exit(1)
1786
1787     def importHeadRevision(self, revision):
1788         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1789
1790         details = {}
1791         details["user"] = "git perforce import user"
1792         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1793                            % (' '.join(self.depotPaths), revision))
1794         details["change"] = revision
1795         newestRevision = 0
1796
1797         fileCnt = 0
1798         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1799
1800         for info in p4CmdList(["files"] + fileArgs):
1801
1802             if 'code' in info and info['code'] == 'error':
1803                 sys.stderr.write("p4 returned an error: %s\n"
1804                                  % info['data'])
1805                 if info['data'].find("must refer to client") >= 0:
1806                     sys.stderr.write("This particular p4 error is misleading.\n")
1807                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1808                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1809                 sys.exit(1)
1810             if 'p4ExitCode' in info:
1811                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1812                 sys.exit(1)
1813
1814
1815             change = int(info["change"])
1816             if change > newestRevision:
1817                 newestRevision = change
1818
1819             if info["action"] in self.delete_actions:
1820                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1821                 #fileCnt = fileCnt + 1
1822                 continue
1823
1824             for prop in ["depotFile", "rev", "action", "type" ]:
1825                 details["%s%s" % (prop, fileCnt)] = info[prop]
1826
1827             fileCnt = fileCnt + 1
1828
1829         details["change"] = newestRevision
1830
1831         # Use time from top-most change so that all git-p4 clones of
1832         # the same p4 repo have the same commit SHA1s.
1833         res = p4CmdList("describe -s %d" % newestRevision)
1834         newestTime = None
1835         for r in res:
1836             if r.has_key('time'):
1837                 newestTime = int(r['time'])
1838         if newestTime is None:
1839             die("\"describe -s\" on newest change %d did not give a time")
1840         details["time"] = newestTime
1841
1842         self.updateOptionDict(details)
1843         try:
1844             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1845         except IOError:
1846             print "IO error with git fast-import. Is your git version recent enough?"
1847             print self.gitError.read()
1848
1849
1850     def getClientSpec(self):
1851         specList = p4CmdList( "client -o" )
1852         temp = {}
1853         for entry in specList:
1854             for k,v in entry.iteritems():
1855                 if k.startswith("View"):
1856
1857                     # p4 has these %%1 to %%9 arguments in specs to
1858                     # reorder paths; which we can't handle (yet :)
1859                     if re.match('%%\d', v) != None:
1860                         print "Sorry, can't handle %%n arguments in client specs"
1861                         sys.exit(1)
1862
1863                     if v.startswith('"'):
1864                         start = 1
1865                     else:
1866                         start = 0
1867                     index = v.find("...")
1868
1869                     # save the "client view"; i.e the RHS of the view
1870                     # line that tells the client where to put the
1871                     # files for this view.
1872                     cv = v[index+3:].strip() # +3 to remove previous '...'
1873
1874                     # if the client view doesn't end with a
1875                     # ... wildcard, then we're going to mess up the
1876                     # output directory, so fail gracefully.
1877                     if not cv.endswith('...'):
1878                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1879                         sys.exit(1)
1880                     cv=cv[:-3]
1881
1882                     # now save the view; +index means included, -index
1883                     # means it should be filtered out.
1884                     v = v[start:index]
1885                     if v.startswith("-"):
1886                         v = v[1:]
1887                         include = -len(v)
1888                     else:
1889                         include = len(v)
1890
1891                     temp[v] = (include, cv)
1892
1893         self.clientSpecDirs = temp.items()
1894         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1895
1896     def run(self, args):
1897         self.depotPaths = []
1898         self.changeRange = ""
1899         self.initialParent = ""
1900         self.previousDepotPaths = []
1901
1902         # map from branch depot path to parent branch
1903         self.knownBranches = {}
1904         self.initialParents = {}
1905         self.hasOrigin = originP4BranchesExist()
1906         if not self.syncWithOrigin:
1907             self.hasOrigin = False
1908
1909         if self.importIntoRemotes:
1910             self.refPrefix = "refs/remotes/p4/"
1911         else:
1912             self.refPrefix = "refs/heads/p4/"
1913
1914         if self.syncWithOrigin and self.hasOrigin:
1915             if not self.silent:
1916                 print "Syncing with origin first by calling git fetch origin"
1917             system("git fetch origin")
1918
1919         if len(self.branch) == 0:
1920             self.branch = self.refPrefix + "master"
1921             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1922                 system("git update-ref %s refs/heads/p4" % self.branch)
1923                 system("git branch -D p4");
1924             # create it /after/ importing, when master exists
1925             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1926                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1927
1928         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1929             self.getClientSpec()
1930
1931         # TODO: should always look at previous commits,
1932         # merge with previous imports, if possible.
1933         if args == []:
1934             if self.hasOrigin:
1935                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1936             self.listExistingP4GitBranches()
1937
1938             if len(self.p4BranchesInGit) > 1:
1939                 if not self.silent:
1940                     print "Importing from/into multiple branches"
1941                 self.detectBranches = True
1942
1943             if self.verbose:
1944                 print "branches: %s" % self.p4BranchesInGit
1945
1946             p4Change = 0
1947             for branch in self.p4BranchesInGit:
1948                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1949
1950                 settings = extractSettingsGitLog(logMsg)
1951
1952                 self.readOptions(settings)
1953                 if (settings.has_key('depot-paths')
1954                     and settings.has_key ('change')):
1955                     change = int(settings['change']) + 1
1956                     p4Change = max(p4Change, change)
1957
1958                     depotPaths = sorted(settings['depot-paths'])
1959                     if self.previousDepotPaths == []:
1960                         self.previousDepotPaths = depotPaths
1961                     else:
1962                         paths = []
1963                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1964                             prev_list = prev.split("/")
1965                             cur_list = cur.split("/")
1966                             for i in range(0, min(len(cur_list), len(prev_list))):
1967                                 if cur_list[i] <> prev_list[i]:
1968                                     i = i - 1
1969                                     break
1970
1971                             paths.append ("/".join(cur_list[:i + 1]))
1972
1973                         self.previousDepotPaths = paths
1974
1975             if p4Change > 0:
1976                 self.depotPaths = sorted(self.previousDepotPaths)
1977                 self.changeRange = "@%s,#head" % p4Change
1978                 if not self.detectBranches:
1979                     self.initialParent = parseRevision(self.branch)
1980                 if not self.silent and not self.detectBranches:
1981                     print "Performing incremental import into %s git branch" % self.branch
1982
1983         if not self.branch.startswith("refs/"):
1984             self.branch = "refs/heads/" + self.branch
1985
1986         if len(args) == 0 and self.depotPaths:
1987             if not self.silent:
1988                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1989         else:
1990             if self.depotPaths and self.depotPaths != args:
1991                 print ("previous import used depot path %s and now %s was specified. "
1992                        "This doesn't work!" % (' '.join (self.depotPaths),
1993                                                ' '.join (args)))
1994                 sys.exit(1)
1995
1996             self.depotPaths = sorted(args)
1997
1998         revision = ""
1999         self.users = {}
2000
2001         newPaths = []
2002         for p in self.depotPaths:
2003             if p.find("@") != -1:
2004                 atIdx = p.index("@")
2005                 self.changeRange = p[atIdx:]
2006                 if self.changeRange == "@all":
2007                     self.changeRange = ""
2008                 elif ',' not in self.changeRange:
2009                     revision = self.changeRange
2010                     self.changeRange = ""
2011                 p = p[:atIdx]
2012             elif p.find("#") != -1:
2013                 hashIdx = p.index("#")
2014                 revision = p[hashIdx:]
2015                 p = p[:hashIdx]
2016             elif self.previousDepotPaths == []:
2017                 revision = "#head"
2018
2019             p = re.sub ("\.\.\.$", "", p)
2020             if not p.endswith("/"):
2021                 p += "/"
2022
2023             newPaths.append(p)
2024
2025         self.depotPaths = newPaths
2026
2027
2028         self.loadUserMapFromCache()
2029         self.labels = {}
2030         if self.detectLabels:
2031             self.getLabels();
2032
2033         if self.detectBranches:
2034             ## FIXME - what's a P4 projectName ?
2035             self.projectName = self.guessProjectName()
2036
2037             if self.hasOrigin:
2038                 self.getBranchMappingFromGitBranches()
2039             else:
2040                 self.getBranchMapping()
2041             if self.verbose:
2042                 print "p4-git branches: %s" % self.p4BranchesInGit
2043                 print "initial parents: %s" % self.initialParents
2044             for b in self.p4BranchesInGit:
2045                 if b != "master":
2046
2047                     ## FIXME
2048                     b = b[len(self.projectName):]
2049                 self.createdBranches.add(b)
2050
2051         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2052
2053         importProcess = subprocess.Popen(["git", "fast-import"],
2054                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2055                                          stderr=subprocess.PIPE);
2056         self.gitOutput = importProcess.stdout
2057         self.gitStream = importProcess.stdin
2058         self.gitError = importProcess.stderr
2059
2060         if revision:
2061             self.importHeadRevision(revision)
2062         else:
2063             changes = []
2064
2065             if len(self.changesFile) > 0:
2066                 output = open(self.changesFile).readlines()
2067                 changeSet = set()
2068                 for line in output:
2069                     changeSet.add(int(line))
2070
2071                 for change in changeSet:
2072                     changes.append(change)
2073
2074                 changes.sort()
2075             else:
2076                 # catch "git-p4 sync" with no new branches, in a repo that
2077                 # does not have any existing git-p4 branches
2078                 if len(args) == 0 and not self.p4BranchesInGit:
2079                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2080                 if self.verbose:
2081                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2082                                                               self.changeRange)
2083                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2084
2085                 if len(self.maxChanges) > 0:
2086                     changes = changes[:min(int(self.maxChanges), len(changes))]
2087
2088             if len(changes) == 0:
2089                 if not self.silent:
2090                     print "No changes to import!"
2091                 return True
2092
2093             if not self.silent and not self.detectBranches:
2094                 print "Import destination: %s" % self.branch
2095
2096             self.updatedBranches = set()
2097
2098             self.importChanges(changes)
2099
2100             if not self.silent:
2101                 print ""
2102                 if len(self.updatedBranches) > 0:
2103                     sys.stdout.write("Updated branches: ")
2104                     for b in self.updatedBranches:
2105                         sys.stdout.write("%s " % b)
2106                     sys.stdout.write("\n")
2107
2108         self.gitStream.close()
2109         if importProcess.wait() != 0:
2110             die("fast-import failed: %s" % self.gitError.read())
2111         self.gitOutput.close()
2112         self.gitError.close()
2113
2114         return True
2115
2116 class P4Rebase(Command):
2117     def __init__(self):
2118         Command.__init__(self)
2119         self.options = [ ]
2120         self.description = ("Fetches the latest revision from perforce and "
2121                             + "rebases the current work (branch) against it")
2122         self.verbose = False
2123
2124     def run(self, args):
2125         sync = P4Sync()
2126         sync.run([])
2127
2128         return self.rebase()
2129
2130     def rebase(self):
2131         if os.system("git update-index --refresh") != 0:
2132             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2133         if len(read_pipe("git diff-index HEAD --")) > 0:
2134             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2135
2136         [upstream, settings] = findUpstreamBranchPoint()
2137         if len(upstream) == 0:
2138             die("Cannot find upstream branchpoint for rebase")
2139
2140         # the branchpoint may be p4/foo~3, so strip off the parent
2141         upstream = re.sub("~[0-9]+$", "", upstream)
2142
2143         print "Rebasing the current branch onto %s" % upstream
2144         oldHead = read_pipe("git rev-parse HEAD").strip()
2145         system("git rebase %s" % upstream)
2146         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2147         return True
2148
2149 class P4Clone(P4Sync):
2150     def __init__(self):
2151         P4Sync.__init__(self)
2152         self.description = "Creates a new git repository and imports from Perforce into it"
2153         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2154         self.options += [
2155             optparse.make_option("--destination", dest="cloneDestination",
2156                                  action='store', default=None,
2157                                  help="where to leave result of the clone"),
2158             optparse.make_option("-/", dest="cloneExclude",
2159                                  action="append", type="string",
2160                                  help="exclude depot path"),
2161             optparse.make_option("--bare", dest="cloneBare",
2162                                  action="store_true", default=False),
2163         ]
2164         self.cloneDestination = None
2165         self.needsGit = False
2166         self.cloneBare = False
2167
2168     # This is required for the "append" cloneExclude action
2169     def ensure_value(self, attr, value):
2170         if not hasattr(self, attr) or getattr(self, attr) is None:
2171             setattr(self, attr, value)
2172         return getattr(self, attr)
2173
2174     def defaultDestination(self, args):
2175         ## TODO: use common prefix of args?
2176         depotPath = args[0]
2177         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2178         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2179         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2180         depotDir = re.sub(r"/$", "", depotDir)
2181         return os.path.split(depotDir)[1]
2182
2183     def run(self, args):
2184         if len(args) < 1:
2185             return False
2186
2187         if self.keepRepoPath and not self.cloneDestination:
2188             sys.stderr.write("Must specify destination for --keep-path\n")
2189             sys.exit(1)
2190
2191         depotPaths = args
2192
2193         if not self.cloneDestination and len(depotPaths) > 1:
2194             self.cloneDestination = depotPaths[-1]
2195             depotPaths = depotPaths[:-1]
2196
2197         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2198         for p in depotPaths:
2199             if not p.startswith("//"):
2200                 return False
2201
2202         if not self.cloneDestination:
2203             self.cloneDestination = self.defaultDestination(args)
2204
2205         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2206
2207         if not os.path.exists(self.cloneDestination):
2208             os.makedirs(self.cloneDestination)
2209         chdir(self.cloneDestination)
2210
2211         init_cmd = [ "git", "init" ]
2212         if self.cloneBare:
2213             init_cmd.append("--bare")
2214         subprocess.check_call(init_cmd)
2215
2216         if not P4Sync.run(self, depotPaths):
2217             return False
2218         if self.branch != "master":
2219             if self.importIntoRemotes:
2220                 masterbranch = "refs/remotes/p4/master"
2221             else:
2222                 masterbranch = "refs/heads/p4/master"
2223             if gitBranchExists(masterbranch):
2224                 system("git branch master %s" % masterbranch)
2225                 if not self.cloneBare:
2226                     system("git checkout -f")
2227             else:
2228                 print "Could not detect main branch. No checkout/master branch created."
2229
2230         return True
2231
2232 class P4Branches(Command):
2233     def __init__(self):
2234         Command.__init__(self)
2235         self.options = [ ]
2236         self.description = ("Shows the git branches that hold imports and their "
2237                             + "corresponding perforce depot paths")
2238         self.verbose = False
2239
2240     def run(self, args):
2241         if originP4BranchesExist():
2242             createOrUpdateBranchesFromOrigin()
2243
2244         cmdline = "git rev-parse --symbolic "
2245         cmdline += " --remotes"
2246
2247         for line in read_pipe_lines(cmdline):
2248             line = line.strip()
2249
2250             if not line.startswith('p4/') or line == "p4/HEAD":
2251                 continue
2252             branch = line
2253
2254             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2255             settings = extractSettingsGitLog(log)
2256
2257             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2258         return True
2259
2260 class HelpFormatter(optparse.IndentedHelpFormatter):
2261     def __init__(self):
2262         optparse.IndentedHelpFormatter.__init__(self)
2263
2264     def format_description(self, description):
2265         if description:
2266             return description + "\n"
2267         else:
2268             return ""
2269
2270 def printUsage(commands):
2271     print "usage: %s <command> [options]" % sys.argv[0]
2272     print ""
2273     print "valid commands: %s" % ", ".join(commands)
2274     print ""
2275     print "Try %s <command> --help for command specific help." % sys.argv[0]
2276     print ""
2277
2278 commands = {
2279     "debug" : P4Debug,
2280     "submit" : P4Submit,
2281     "commit" : P4Submit,
2282     "sync" : P4Sync,
2283     "rebase" : P4Rebase,
2284     "clone" : P4Clone,
2285     "rollback" : P4RollBack,
2286     "branches" : P4Branches
2287 }
2288
2289
2290 def main():
2291     if len(sys.argv[1:]) == 0:
2292         printUsage(commands.keys())
2293         sys.exit(2)
2294
2295     cmd = ""
2296     cmdName = sys.argv[1]
2297     try:
2298         klass = commands[cmdName]
2299         cmd = klass()
2300     except KeyError:
2301         print "unknown command %s" % cmdName
2302         print ""
2303         printUsage(commands.keys())
2304         sys.exit(2)
2305
2306     options = cmd.options
2307     cmd.gitdir = os.environ.get("GIT_DIR", None)
2308
2309     args = sys.argv[2:]
2310
2311     if len(options) > 0:
2312         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2313
2314         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2315                                        options,
2316                                        description = cmd.description,
2317                                        formatter = HelpFormatter())
2318
2319         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2320     global verbose
2321     verbose = cmd.verbose
2322     if cmd.needsGit:
2323         if cmd.gitdir == None:
2324             cmd.gitdir = os.path.abspath(".git")
2325             if not isValidGitDir(cmd.gitdir):
2326                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2327                 if os.path.exists(cmd.gitdir):
2328                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2329                     if len(cdup) > 0:
2330                         chdir(cdup);
2331
2332         if not isValidGitDir(cmd.gitdir):
2333             if isValidGitDir(cmd.gitdir + "/.git"):
2334                 cmd.gitdir += "/.git"
2335             else:
2336                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2337
2338         os.environ["GIT_DIR"] = cmd.gitdir
2339
2340     if not cmd.run(args):
2341         parser.print_help()
2342
2343
2344 if __name__ == '__main__':
2345     main()