3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
27 from subprocess import CalledProcessError
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
46 # Grab changes in blocks of this many revisions, unless otherwise requested
47 defaultBlockSize = 512
49 def p4_build_cmd(cmd):
50 """Build a suitable p4 command line.
52 This consolidates building and returning a p4 command line into one
53 location. It means that hooking into the environment, or other configuration
54 can be done more easily.
58 user = gitConfig("git-p4.user")
60 real_cmd += ["-u",user]
62 password = gitConfig("git-p4.password")
64 real_cmd += ["-P", password]
66 port = gitConfig("git-p4.port")
68 real_cmd += ["-p", port]
70 host = gitConfig("git-p4.host")
72 real_cmd += ["-H", host]
74 client = gitConfig("git-p4.client")
76 real_cmd += ["-c", client]
79 if isinstance(cmd,basestring):
80 real_cmd = ' '.join(real_cmd) + ' ' + cmd
85 def chdir(path, is_client_path=False):
86 """Do chdir to the given path, and set the PWD environment
87 variable for use by P4. It does not look at getcwd() output.
88 Since we're not using the shell, it is necessary to set the
89 PWD environment variable explicitly.
91 Normally, expand the path to force it to be absolute. This
92 addresses the use of relative path names inside P4 settings,
93 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
94 as given; it looks for .p4config using PWD.
96 If is_client_path, the path was handed to us directly by p4,
97 and may be a symbolic link. Do not call os.getcwd() in this
98 case, because it will cause p4 to think that PWD is not inside
103 if not is_client_path:
105 os.environ['PWD'] = path
111 sys.stderr.write(msg + "\n")
114 def write_pipe(c, stdin):
116 sys.stderr.write('Writing pipe: %s\n' % str(c))
118 expand = isinstance(c,basestring)
119 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
121 val = pipe.write(stdin)
124 die('Command failed: %s' % str(c))
128 def p4_write_pipe(c, stdin):
129 real_cmd = p4_build_cmd(c)
130 return write_pipe(real_cmd, stdin)
132 def read_pipe(c, ignore_error=False):
134 sys.stderr.write('Reading pipe: %s\n' % str(c))
136 expand = isinstance(c,basestring)
137 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
138 (out, err) = p.communicate()
139 if p.returncode != 0 and not ignore_error:
140 die('Command failed: %s\nError: %s' % (str(c), err))
143 def p4_read_pipe(c, ignore_error=False):
144 real_cmd = p4_build_cmd(c)
145 return read_pipe(real_cmd, ignore_error)
147 def read_pipe_lines(c):
149 sys.stderr.write('Reading pipe: %s\n' % str(c))
151 expand = isinstance(c, basestring)
152 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
154 val = pipe.readlines()
155 if pipe.close() or p.wait():
156 die('Command failed: %s' % str(c))
160 def p4_read_pipe_lines(c):
161 """Specifically invoke p4 on the command supplied. """
162 real_cmd = p4_build_cmd(c)
163 return read_pipe_lines(real_cmd)
165 def p4_has_command(cmd):
166 """Ask p4 for help on this command. If it returns an error, the
167 command does not exist in this version of p4."""
168 real_cmd = p4_build_cmd(["help", cmd])
169 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
170 stderr=subprocess.PIPE)
172 return p.returncode == 0
174 def p4_has_move_command():
175 """See if the move command exists, that it supports -k, and that
176 it has not been administratively disabled. The arguments
177 must be correct, but the filenames do not have to exist. Use
178 ones with wildcards so even if they exist, it will fail."""
180 if not p4_has_command("move"):
182 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
183 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
184 (out, err) = p.communicate()
185 # return code will be 1 in either case
186 if err.find("Invalid option") >= 0:
188 if err.find("disabled") >= 0:
190 # assume it failed because @... was invalid changelist
194 expand = isinstance(cmd,basestring)
196 sys.stderr.write("executing %s\n" % str(cmd))
197 retcode = subprocess.call(cmd, shell=expand)
199 raise CalledProcessError(retcode, cmd)
202 """Specifically invoke p4 as the system command. """
203 real_cmd = p4_build_cmd(cmd)
204 expand = isinstance(real_cmd, basestring)
205 retcode = subprocess.call(real_cmd, shell=expand)
207 raise CalledProcessError(retcode, real_cmd)
209 _p4_version_string = None
210 def p4_version_string():
211 """Read the version string, showing just the last line, which
212 hopefully is the interesting version bit.
215 Perforce - The Fast Software Configuration Management System.
216 Copyright 1995-2011 Perforce Software. All rights reserved.
217 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
219 global _p4_version_string
220 if not _p4_version_string:
221 a = p4_read_pipe_lines(["-V"])
222 _p4_version_string = a[-1].rstrip()
223 return _p4_version_string
225 def p4_integrate(src, dest):
226 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
228 def p4_sync(f, *options):
229 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
232 # forcibly add file names with wildcards
233 if wildcard_present(f):
234 p4_system(["add", "-f", f])
236 p4_system(["add", f])
239 p4_system(["delete", wildcard_encode(f)])
242 p4_system(["edit", wildcard_encode(f)])
245 p4_system(["revert", wildcard_encode(f)])
247 def p4_reopen(type, f):
248 p4_system(["reopen", "-t", type, wildcard_encode(f)])
250 def p4_move(src, dest):
251 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
253 def p4_last_change():
254 results = p4CmdList(["changes", "-m", "1"])
255 return int(results[0]['change'])
257 def p4_describe(change):
258 """Make sure it returns a valid result by checking for
259 the presence of field "time". Return a dict of the
262 ds = p4CmdList(["describe", "-s", str(change)])
264 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
268 if "p4ExitCode" in d:
269 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
272 if d["code"] == "error":
273 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
276 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
281 # Canonicalize the p4 type and return a tuple of the
282 # base type, plus any modifiers. See "p4 help filetypes"
283 # for a list and explanation.
285 def split_p4_type(p4type):
287 p4_filetypes_historical = {
288 "ctempobj": "binary+Sw",
294 "tempobj": "binary+FSw",
295 "ubinary": "binary+F",
296 "uresource": "resource+F",
297 "uxbinary": "binary+Fx",
298 "xbinary": "binary+x",
300 "xtempobj": "binary+Swx",
302 "xunicode": "unicode+x",
305 if p4type in p4_filetypes_historical:
306 p4type = p4_filetypes_historical[p4type]
308 s = p4type.split("+")
316 # return the raw p4 type of a file (text, text+ko, etc)
319 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
320 return results[0]['headType']
323 # Given a type base and modifier, return a regexp matching
324 # the keywords that can be expanded in the file
326 def p4_keywords_regexp_for_type(base, type_mods):
327 if base in ("text", "unicode", "binary"):
329 if "ko" in type_mods:
331 elif "k" in type_mods:
332 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
336 \$ # Starts with a dollar, followed by...
337 (%s) # one of the keywords, followed by...
338 (:[^$\n]+)? # possibly an old expansion, followed by...
346 # Given a file, return a regexp matching the possible
347 # RCS keywords that will be expanded, or None for files
348 # with kw expansion turned off.
350 def p4_keywords_regexp_for_file(file):
351 if not os.path.exists(file):
354 (type_base, type_mods) = split_p4_type(p4_type(file))
355 return p4_keywords_regexp_for_type(type_base, type_mods)
357 def setP4ExecBit(file, mode):
358 # Reopens an already open file and changes the execute bit to match
359 # the execute bit setting in the passed in mode.
363 if not isModeExec(mode):
364 p4Type = getP4OpenedType(file)
365 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
366 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
367 if p4Type[-1] == "+":
368 p4Type = p4Type[0:-1]
370 p4_reopen(p4Type, file)
372 def getP4OpenedType(file):
373 # Returns the perforce file type for the given file.
375 result = p4_read_pipe(["opened", wildcard_encode(file)])
376 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
378 return match.group(1)
380 die("Could not determine file type for %s (result: '%s')" % (file, result))
382 # Return the set of all p4 labels
383 def getP4Labels(depotPaths):
385 if isinstance(depotPaths,basestring):
386 depotPaths = [depotPaths]
388 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
394 # Return the set of all git tags
397 for line in read_pipe_lines(["git", "tag"]):
402 def diffTreePattern():
403 # This is a simple generator for the diff tree regex pattern. This could be
404 # a class variable if this and parseDiffTreeEntry were a part of a class.
405 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
409 def parseDiffTreeEntry(entry):
410 """Parses a single diff tree entry into its component elements.
412 See git-diff-tree(1) manpage for details about the format of the diff
413 output. This method returns a dictionary with the following elements:
415 src_mode - The mode of the source file
416 dst_mode - The mode of the destination file
417 src_sha1 - The sha1 for the source file
418 dst_sha1 - The sha1 fr the destination file
419 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
420 status_score - The score for the status (applicable for 'C' and 'R'
421 statuses). This is None if there is no score.
422 src - The path for the source file.
423 dst - The path for the destination file. This is only present for
424 copy or renames. If it is not present, this is None.
426 If the pattern is not matched, None is returned."""
428 match = diffTreePattern().next().match(entry)
431 'src_mode': match.group(1),
432 'dst_mode': match.group(2),
433 'src_sha1': match.group(3),
434 'dst_sha1': match.group(4),
435 'status': match.group(5),
436 'status_score': match.group(6),
437 'src': match.group(7),
438 'dst': match.group(10)
442 def isModeExec(mode):
443 # Returns True if the given git mode represents an executable file,
445 return mode[-3:] == "755"
447 def isModeExecChanged(src_mode, dst_mode):
448 return isModeExec(src_mode) != isModeExec(dst_mode)
450 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
452 if isinstance(cmd,basestring):
459 cmd = p4_build_cmd(cmd)
461 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
463 # Use a temporary file to avoid deadlocks without
464 # subprocess.communicate(), which would put another copy
465 # of stdout into memory.
467 if stdin is not None:
468 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
469 if isinstance(stdin,basestring):
470 stdin_file.write(stdin)
473 stdin_file.write(i + '\n')
477 p4 = subprocess.Popen(cmd,
480 stdout=subprocess.PIPE)
485 entry = marshal.load(p4.stdout)
495 entry["p4ExitCode"] = exitCode
501 list = p4CmdList(cmd)
507 def p4Where(depotPath):
508 if not depotPath.endswith("/"):
510 depotPathLong = depotPath + "..."
511 outputList = p4CmdList(["where", depotPathLong])
513 for entry in outputList:
514 if "depotFile" in entry:
515 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
516 # The base path always ends with "/...".
517 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
520 elif "data" in entry:
521 data = entry.get("data")
522 space = data.find(" ")
523 if data[:space] == depotPath:
528 if output["code"] == "error":
532 clientPath = output.get("path")
533 elif "data" in output:
534 data = output.get("data")
535 lastSpace = data.rfind(" ")
536 clientPath = data[lastSpace + 1:]
538 if clientPath.endswith("..."):
539 clientPath = clientPath[:-3]
542 def currentGitBranch():
543 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
545 def isValidGitDir(path):
546 if (os.path.exists(path + "/HEAD")
547 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
551 def parseRevision(ref):
552 return read_pipe("git rev-parse %s" % ref).strip()
554 def branchExists(ref):
555 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
559 def extractLogMessageFromGitCommit(commit):
562 ## fixme: title is first line of commit, not 1st paragraph.
564 for log in read_pipe_lines("git cat-file commit %s" % commit):
573 def extractSettingsGitLog(log):
575 for line in log.split("\n"):
577 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
581 assignments = m.group(1).split (':')
582 for a in assignments:
584 key = vals[0].strip()
585 val = ('='.join (vals[1:])).strip()
586 if val.endswith ('\"') and val.startswith('"'):
591 paths = values.get("depot-paths")
593 paths = values.get("depot-path")
595 values['depot-paths'] = paths.split(',')
598 def gitBranchExists(branch):
599 proc = subprocess.Popen(["git", "rev-parse", branch],
600 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
601 return proc.wait() == 0;
606 if not _gitConfig.has_key(key):
607 cmd = [ "git", "config", key ]
608 s = read_pipe(cmd, ignore_error=True)
609 _gitConfig[key] = s.strip()
610 return _gitConfig[key]
612 def gitConfigBool(key):
613 """Return a bool, using git config --bool. It is True only if the
614 variable is set to true, and False if set to false or not present
617 if not _gitConfig.has_key(key):
618 cmd = [ "git", "config", "--bool", key ]
619 s = read_pipe(cmd, ignore_error=True)
621 _gitConfig[key] = v == "true"
622 return _gitConfig[key]
624 def gitConfigList(key):
625 if not _gitConfig.has_key(key):
626 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
627 _gitConfig[key] = s.strip().split(os.linesep)
628 return _gitConfig[key]
630 def p4BranchesInGit(branchesAreInRemotes=True):
631 """Find all the branches whose names start with "p4/", looking
632 in remotes or heads as specified by the argument. Return
633 a dictionary of { branch: revision } for each one found.
634 The branch names are the short names, without any
639 cmdline = "git rev-parse --symbolic "
640 if branchesAreInRemotes:
641 cmdline += "--remotes"
643 cmdline += "--branches"
645 for line in read_pipe_lines(cmdline):
649 if not line.startswith('p4/'):
651 # special symbolic ref to p4/master
652 if line == "p4/HEAD":
655 # strip off p4/ prefix
656 branch = line[len("p4/"):]
658 branches[branch] = parseRevision(line)
662 def branch_exists(branch):
663 """Make sure that the given ref name really exists."""
665 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
666 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
667 out, _ = p.communicate()
670 # expect exactly one line of output: the branch name
671 return out.rstrip() == branch
673 def findUpstreamBranchPoint(head = "HEAD"):
674 branches = p4BranchesInGit()
675 # map from depot-path to branch name
676 branchByDepotPath = {}
677 for branch in branches.keys():
678 tip = branches[branch]
679 log = extractLogMessageFromGitCommit(tip)
680 settings = extractSettingsGitLog(log)
681 if settings.has_key("depot-paths"):
682 paths = ",".join(settings["depot-paths"])
683 branchByDepotPath[paths] = "remotes/p4/" + branch
687 while parent < 65535:
688 commit = head + "~%s" % parent
689 log = extractLogMessageFromGitCommit(commit)
690 settings = extractSettingsGitLog(log)
691 if settings.has_key("depot-paths"):
692 paths = ",".join(settings["depot-paths"])
693 if branchByDepotPath.has_key(paths):
694 return [branchByDepotPath[paths], settings]
698 return ["", settings]
700 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
702 print ("Creating/updating branch(es) in %s based on origin branch(es)"
705 originPrefix = "origin/p4/"
707 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
709 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
712 headName = line[len(originPrefix):]
713 remoteHead = localRefPrefix + headName
716 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
717 if (not original.has_key('depot-paths')
718 or not original.has_key('change')):
722 if not gitBranchExists(remoteHead):
724 print "creating %s" % remoteHead
727 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
728 if settings.has_key('change') > 0:
729 if settings['depot-paths'] == original['depot-paths']:
730 originP4Change = int(original['change'])
731 p4Change = int(settings['change'])
732 if originP4Change > p4Change:
733 print ("%s (%s) is newer than %s (%s). "
734 "Updating p4 branch from origin."
735 % (originHead, originP4Change,
736 remoteHead, p4Change))
739 print ("Ignoring: %s was imported from %s while "
740 "%s was imported from %s"
741 % (originHead, ','.join(original['depot-paths']),
742 remoteHead, ','.join(settings['depot-paths'])))
745 system("git update-ref %s %s" % (remoteHead, originHead))
747 def originP4BranchesExist():
748 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
751 def p4ParseNumericChangeRange(parts):
752 changeStart = int(parts[0][1:])
753 if parts[1] == '#head':
754 changeEnd = p4_last_change()
756 changeEnd = int(parts[1])
758 return (changeStart, changeEnd)
760 def chooseBlockSize(blockSize):
764 return defaultBlockSize
766 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
769 # Parse the change range into start and end. Try to find integer
770 # revision ranges as these can be broken up into blocks to avoid
771 # hitting server-side limits (maxrows, maxscanresults). But if
772 # that doesn't work, fall back to using the raw revision specifier
773 # strings, without using block mode.
775 if changeRange is None or changeRange == '':
777 changeEnd = p4_last_change()
778 block_size = chooseBlockSize(requestedBlockSize)
780 parts = changeRange.split(',')
781 assert len(parts) == 2
783 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
784 block_size = chooseBlockSize(requestedBlockSize)
786 changeStart = parts[0][1:]
788 if requestedBlockSize:
789 die("cannot use --changes-block-size with non-numeric revisions")
792 # Accumulate change numbers in a dictionary to avoid duplicates
796 # Retrieve changes a block at a time, to prevent running
797 # into a MaxResults/MaxScanRows error from the server.
803 end = min(changeEnd, changeStart + block_size)
804 revisionRange = "%d,%d" % (changeStart, end)
806 revisionRange = "%s,%s" % (changeStart, changeEnd)
808 cmd += ["%s...@%s" % (p, revisionRange)]
810 for line in p4_read_pipe_lines(cmd):
811 changeNum = int(line.split(" ")[1])
812 changes[changeNum] = True
820 changeStart = end + 1
822 changelist = changes.keys()
826 def p4PathStartsWith(path, prefix):
827 # This method tries to remedy a potential mixed-case issue:
829 # If UserA adds //depot/DirA/file1
830 # and UserB adds //depot/dira/file2
832 # we may or may not have a problem. If you have core.ignorecase=true,
833 # we treat DirA and dira as the same directory
834 if gitConfigBool("core.ignorecase"):
835 return path.lower().startswith(prefix.lower())
836 return path.startswith(prefix)
839 """Look at the p4 client spec, create a View() object that contains
840 all the mappings, and return it."""
842 specList = p4CmdList("client -o")
843 if len(specList) != 1:
844 die('Output from "client -o" is %d lines, expecting 1' %
847 # dictionary of all client parameters
851 client_name = entry["Client"]
853 # just the keys that start with "View"
854 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
857 view = View(client_name)
859 # append the lines, in order, to the view
860 for view_num in range(len(view_keys)):
861 k = "View%d" % view_num
862 if k not in view_keys:
863 die("Expected view key %s missing" % k)
864 view.append(entry[k])
869 """Grab the client directory."""
871 output = p4CmdList("client -o")
873 die('Output from "client -o" is %d lines, expecting 1' % len(output))
876 if "Root" not in entry:
877 die('Client has no "Root"')
882 # P4 wildcards are not allowed in filenames. P4 complains
883 # if you simply add them, but you can force it with "-f", in
884 # which case it translates them into %xx encoding internally.
886 def wildcard_decode(path):
887 # Search for and fix just these four characters. Do % last so
888 # that fixing it does not inadvertently create new %-escapes.
889 # Cannot have * in a filename in windows; untested as to
890 # what p4 would do in such a case.
891 if not platform.system() == "Windows":
892 path = path.replace("%2A", "*")
893 path = path.replace("%23", "#") \
894 .replace("%40", "@") \
898 def wildcard_encode(path):
899 # do % first to avoid double-encoding the %s introduced here
900 path = path.replace("%", "%25") \
901 .replace("*", "%2A") \
902 .replace("#", "%23") \
906 def wildcard_present(path):
907 m = re.search("[*#@%]", path)
912 self.usage = "usage: %prog [options]"
918 self.userMapFromPerforceServer = False
919 self.myP4UserId = None
923 return self.myP4UserId
925 results = p4CmdList("user -o")
927 if r.has_key('User'):
928 self.myP4UserId = r['User']
930 die("Could not find your p4 user id")
932 def p4UserIsMe(self, p4User):
933 # return True if the given p4 user is actually me
935 if not p4User or p4User != me:
940 def getUserCacheFilename(self):
941 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
942 return home + "/.gitp4-usercache.txt"
944 def getUserMapFromPerforceServer(self):
945 if self.userMapFromPerforceServer:
950 for output in p4CmdList("users"):
951 if not output.has_key("User"):
953 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
954 self.emails[output["Email"]] = output["User"]
958 for (key, val) in self.users.items():
959 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
961 open(self.getUserCacheFilename(), "wb").write(s)
962 self.userMapFromPerforceServer = True
964 def loadUserMapFromCache(self):
966 self.userMapFromPerforceServer = False
968 cache = open(self.getUserCacheFilename(), "rb")
969 lines = cache.readlines()
972 entry = line.strip().split("\t")
973 self.users[entry[0]] = entry[1]
975 self.getUserMapFromPerforceServer()
977 class P4Debug(Command):
979 Command.__init__(self)
981 self.description = "A tool to debug the output of p4 -G."
982 self.needsGit = False
986 for output in p4CmdList(args):
987 print 'Element: %d' % j
992 class P4RollBack(Command):
994 Command.__init__(self)
996 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
998 self.description = "A tool to debug the multi-branch import. Don't use :)"
999 self.rollbackLocalBranches = False
1001 def run(self, args):
1004 maxChange = int(args[0])
1006 if "p4ExitCode" in p4Cmd("changes -m 1"):
1007 die("Problems executing p4");
1009 if self.rollbackLocalBranches:
1010 refPrefix = "refs/heads/"
1011 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1013 refPrefix = "refs/remotes/"
1014 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1017 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1019 ref = refPrefix + line
1020 log = extractLogMessageFromGitCommit(ref)
1021 settings = extractSettingsGitLog(log)
1023 depotPaths = settings['depot-paths']
1024 change = settings['change']
1028 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1029 for p in depotPaths]))) == 0:
1030 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1031 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1034 while change and int(change) > maxChange:
1037 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1038 system("git update-ref %s \"%s^\"" % (ref, ref))
1039 log = extractLogMessageFromGitCommit(ref)
1040 settings = extractSettingsGitLog(log)
1043 depotPaths = settings['depot-paths']
1044 change = settings['change']
1047 print "%s rewound to %s" % (ref, change)
1051 class P4Submit(Command, P4UserMap):
1053 conflict_behavior_choices = ("ask", "skip", "quit")
1056 Command.__init__(self)
1057 P4UserMap.__init__(self)
1059 optparse.make_option("--origin", dest="origin"),
1060 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1061 # preserve the user, requires relevant p4 permissions
1062 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1063 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1064 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1065 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1066 optparse.make_option("--conflict", dest="conflict_behavior",
1067 choices=self.conflict_behavior_choices),
1068 optparse.make_option("--branch", dest="branch"),
1070 self.description = "Submit changes from git to the perforce depot."
1071 self.usage += " [name of git branch to submit into perforce depot]"
1073 self.detectRenames = False
1074 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1075 self.dry_run = False
1076 self.prepare_p4_only = False
1077 self.conflict_behavior = None
1078 self.isWindows = (platform.system() == "Windows")
1079 self.exportLabels = False
1080 self.p4HasMoveCommand = p4_has_move_command()
1084 if len(p4CmdList("opened ...")) > 0:
1085 die("You have files opened with perforce! Close them before starting the sync.")
1087 def separate_jobs_from_description(self, message):
1088 """Extract and return a possible Jobs field in the commit
1089 message. It goes into a separate section in the p4 change
1092 A jobs line starts with "Jobs:" and looks like a new field
1093 in a form. Values are white-space separated on the same
1094 line or on following lines that start with a tab.
1096 This does not parse and extract the full git commit message
1097 like a p4 form. It just sees the Jobs: line as a marker
1098 to pass everything from then on directly into the p4 form,
1099 but outside the description section.
1101 Return a tuple (stripped log message, jobs string)."""
1103 m = re.search(r'^Jobs:', message, re.MULTILINE)
1105 return (message, None)
1107 jobtext = message[m.start():]
1108 stripped_message = message[:m.start()].rstrip()
1109 return (stripped_message, jobtext)
1111 def prepareLogMessage(self, template, message, jobs):
1112 """Edits the template returned from "p4 change -o" to insert
1113 the message in the Description field, and the jobs text in
1117 inDescriptionSection = False
1119 for line in template.split("\n"):
1120 if line.startswith("#"):
1121 result += line + "\n"
1124 if inDescriptionSection:
1125 if line.startswith("Files:") or line.startswith("Jobs:"):
1126 inDescriptionSection = False
1127 # insert Jobs section
1129 result += jobs + "\n"
1133 if line.startswith("Description:"):
1134 inDescriptionSection = True
1136 for messageLine in message.split("\n"):
1137 line += "\t" + messageLine + "\n"
1139 result += line + "\n"
1143 def patchRCSKeywords(self, file, pattern):
1144 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1145 (handle, outFileName) = tempfile.mkstemp(dir='.')
1147 outFile = os.fdopen(handle, "w+")
1148 inFile = open(file, "r")
1149 regexp = re.compile(pattern, re.VERBOSE)
1150 for line in inFile.readlines():
1151 line = regexp.sub(r'$\1$', line)
1155 # Forcibly overwrite the original file
1157 shutil.move(outFileName, file)
1159 # cleanup our temporary file
1160 os.unlink(outFileName)
1161 print "Failed to strip RCS keywords in %s" % file
1164 print "Patched up RCS keywords in %s" % file
1166 def p4UserForCommit(self,id):
1167 # Return the tuple (perforce user,git email) for a given git commit id
1168 self.getUserMapFromPerforceServer()
1169 gitEmail = read_pipe(["git", "log", "--max-count=1",
1170 "--format=%ae", id])
1171 gitEmail = gitEmail.strip()
1172 if not self.emails.has_key(gitEmail):
1173 return (None,gitEmail)
1175 return (self.emails[gitEmail],gitEmail)
1177 def checkValidP4Users(self,commits):
1178 # check if any git authors cannot be mapped to p4 users
1180 (user,email) = self.p4UserForCommit(id)
1182 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1183 if gitConfigBool("git-p4.allowMissingP4Users"):
1186 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1188 def lastP4Changelist(self):
1189 # Get back the last changelist number submitted in this client spec. This
1190 # then gets used to patch up the username in the change. If the same
1191 # client spec is being used by multiple processes then this might go
1193 results = p4CmdList("client -o") # find the current client
1196 if r.has_key('Client'):
1197 client = r['Client']
1200 die("could not get client spec")
1201 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1203 if r.has_key('change'):
1205 die("Could not get changelist number for last submit - cannot patch up user details")
1207 def modifyChangelistUser(self, changelist, newUser):
1208 # fixup the user field of a changelist after it has been submitted.
1209 changes = p4CmdList("change -o %s" % changelist)
1210 if len(changes) != 1:
1211 die("Bad output from p4 change modifying %s to user %s" %
1212 (changelist, newUser))
1215 if c['User'] == newUser: return # nothing to do
1217 input = marshal.dumps(c)
1219 result = p4CmdList("change -f -i", stdin=input)
1221 if r.has_key('code'):
1222 if r['code'] == 'error':
1223 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1224 if r.has_key('data'):
1225 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1227 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1229 def canChangeChangelists(self):
1230 # check to see if we have p4 admin or super-user permissions, either of
1231 # which are required to modify changelists.
1232 results = p4CmdList(["protects", self.depotPath])
1234 if r.has_key('perm'):
1235 if r['perm'] == 'admin':
1237 if r['perm'] == 'super':
1241 def prepareSubmitTemplate(self):
1242 """Run "p4 change -o" to grab a change specification template.
1243 This does not use "p4 -G", as it is nice to keep the submission
1244 template in original order, since a human might edit it.
1246 Remove lines in the Files section that show changes to files
1247 outside the depot path we're committing into."""
1250 inFilesSection = False
1251 for line in p4_read_pipe_lines(['change', '-o']):
1252 if line.endswith("\r\n"):
1253 line = line[:-2] + "\n"
1255 if line.startswith("\t"):
1256 # path starts and ends with a tab
1258 lastTab = path.rfind("\t")
1260 path = path[:lastTab]
1261 if not p4PathStartsWith(path, self.depotPath):
1264 inFilesSection = False
1266 if line.startswith("Files:"):
1267 inFilesSection = True
1273 def edit_template(self, template_file):
1274 """Invoke the editor to let the user change the submission
1275 message. Return true if okay to continue with the submit."""
1277 # if configured to skip the editing part, just submit
1278 if gitConfigBool("git-p4.skipSubmitEdit"):
1281 # look at the modification time, to check later if the user saved
1283 mtime = os.stat(template_file).st_mtime
1286 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1287 editor = os.environ.get("P4EDITOR")
1289 editor = read_pipe("git var GIT_EDITOR").strip()
1290 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1292 # If the file was not saved, prompt to see if this patch should
1293 # be skipped. But skip this verification step if configured so.
1294 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1297 # modification time updated means user saved the file
1298 if os.stat(template_file).st_mtime > mtime:
1302 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1308 def get_diff_description(self, editedFiles, filesToAdd):
1310 if os.environ.has_key("P4DIFF"):
1311 del(os.environ["P4DIFF"])
1313 for editedFile in editedFiles:
1314 diff += p4_read_pipe(['diff', '-du',
1315 wildcard_encode(editedFile)])
1319 for newFile in filesToAdd:
1320 newdiff += "==== new file ====\n"
1321 newdiff += "--- /dev/null\n"
1322 newdiff += "+++ %s\n" % newFile
1323 f = open(newFile, "r")
1324 for line in f.readlines():
1325 newdiff += "+" + line
1328 return (diff + newdiff).replace('\r\n', '\n')
1330 def applyCommit(self, id):
1331 """Apply one commit, return True if it succeeded."""
1333 print "Applying", read_pipe(["git", "show", "-s",
1334 "--format=format:%h %s", id])
1336 (p4User, gitEmail) = self.p4UserForCommit(id)
1338 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1340 filesToDelete = set()
1342 pureRenameCopy = set()
1343 filesToChangeExecBit = {}
1346 diff = parseDiffTreeEntry(line)
1347 modifier = diff['status']
1351 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1352 filesToChangeExecBit[path] = diff['dst_mode']
1353 editedFiles.add(path)
1354 elif modifier == "A":
1355 filesToAdd.add(path)
1356 filesToChangeExecBit[path] = diff['dst_mode']
1357 if path in filesToDelete:
1358 filesToDelete.remove(path)
1359 elif modifier == "D":
1360 filesToDelete.add(path)
1361 if path in filesToAdd:
1362 filesToAdd.remove(path)
1363 elif modifier == "C":
1364 src, dest = diff['src'], diff['dst']
1365 p4_integrate(src, dest)
1366 pureRenameCopy.add(dest)
1367 if diff['src_sha1'] != diff['dst_sha1']:
1369 pureRenameCopy.discard(dest)
1370 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1372 pureRenameCopy.discard(dest)
1373 filesToChangeExecBit[dest] = diff['dst_mode']
1375 # turn off read-only attribute
1376 os.chmod(dest, stat.S_IWRITE)
1378 editedFiles.add(dest)
1379 elif modifier == "R":
1380 src, dest = diff['src'], diff['dst']
1381 if self.p4HasMoveCommand:
1382 p4_edit(src) # src must be open before move
1383 p4_move(src, dest) # opens for (move/delete, move/add)
1385 p4_integrate(src, dest)
1386 if diff['src_sha1'] != diff['dst_sha1']:
1389 pureRenameCopy.add(dest)
1390 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1391 if not self.p4HasMoveCommand:
1392 p4_edit(dest) # with move: already open, writable
1393 filesToChangeExecBit[dest] = diff['dst_mode']
1394 if not self.p4HasMoveCommand:
1396 os.chmod(dest, stat.S_IWRITE)
1398 filesToDelete.add(src)
1399 editedFiles.add(dest)
1401 die("unknown modifier %s for %s" % (modifier, path))
1403 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1404 patchcmd = diffcmd + " | git apply "
1405 tryPatchCmd = patchcmd + "--check -"
1406 applyPatchCmd = patchcmd + "--check --apply -"
1407 patch_succeeded = True
1409 if os.system(tryPatchCmd) != 0:
1410 fixed_rcs_keywords = False
1411 patch_succeeded = False
1412 print "Unfortunately applying the change failed!"
1414 # Patch failed, maybe it's just RCS keyword woes. Look through
1415 # the patch to see if that's possible.
1416 if gitConfigBool("git-p4.attemptRCSCleanup"):
1420 for file in editedFiles | filesToDelete:
1421 # did this file's delta contain RCS keywords?
1422 pattern = p4_keywords_regexp_for_file(file)
1425 # this file is a possibility...look for RCS keywords.
1426 regexp = re.compile(pattern, re.VERBOSE)
1427 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1428 if regexp.search(line):
1430 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1431 kwfiles[file] = pattern
1434 for file in kwfiles:
1436 print "zapping %s with %s" % (line,pattern)
1437 # File is being deleted, so not open in p4. Must
1438 # disable the read-only bit on windows.
1439 if self.isWindows and file not in editedFiles:
1440 os.chmod(file, stat.S_IWRITE)
1441 self.patchRCSKeywords(file, kwfiles[file])
1442 fixed_rcs_keywords = True
1444 if fixed_rcs_keywords:
1445 print "Retrying the patch with RCS keywords cleaned up"
1446 if os.system(tryPatchCmd) == 0:
1447 patch_succeeded = True
1449 if not patch_succeeded:
1450 for f in editedFiles:
1455 # Apply the patch for real, and do add/delete/+x handling.
1457 system(applyPatchCmd)
1459 for f in filesToAdd:
1461 for f in filesToDelete:
1465 # Set/clear executable bits
1466 for f in filesToChangeExecBit.keys():
1467 mode = filesToChangeExecBit[f]
1468 setP4ExecBit(f, mode)
1471 # Build p4 change description, starting with the contents
1472 # of the git commit message.
1474 logMessage = extractLogMessageFromGitCommit(id)
1475 logMessage = logMessage.strip()
1476 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1478 template = self.prepareSubmitTemplate()
1479 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1481 if self.preserveUser:
1482 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1484 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1485 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1486 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1487 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1489 separatorLine = "######## everything below this line is just the diff #######\n"
1490 if not self.prepare_p4_only:
1491 submitTemplate += separatorLine
1492 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1494 (handle, fileName) = tempfile.mkstemp()
1495 tmpFile = os.fdopen(handle, "w+b")
1497 submitTemplate = submitTemplate.replace("\n", "\r\n")
1498 tmpFile.write(submitTemplate)
1501 if self.prepare_p4_only:
1503 # Leave the p4 tree prepared, and the submit template around
1504 # and let the user decide what to do next
1507 print "P4 workspace prepared for submission."
1508 print "To submit or revert, go to client workspace"
1509 print " " + self.clientPath
1511 print "To submit, use \"p4 submit\" to write a new description,"
1512 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1513 " \"git p4\"." % fileName
1514 print "You can delete the file \"%s\" when finished." % fileName
1516 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1517 print "To preserve change ownership by user %s, you must\n" \
1518 "do \"p4 change -f <change>\" after submitting and\n" \
1519 "edit the User field."
1521 print "After submitting, renamed files must be re-synced."
1522 print "Invoke \"p4 sync -f\" on each of these files:"
1523 for f in pureRenameCopy:
1527 print "To revert the changes, use \"p4 revert ...\", and delete"
1528 print "the submit template file \"%s\"" % fileName
1530 print "Since the commit adds new files, they must be deleted:"
1531 for f in filesToAdd:
1537 # Let the user edit the change description, then submit it.
1539 if self.edit_template(fileName):
1540 # read the edited message and submit
1542 tmpFile = open(fileName, "rb")
1543 message = tmpFile.read()
1546 message = message.replace("\r\n", "\n")
1547 submitTemplate = message[:message.index(separatorLine)]
1548 p4_write_pipe(['submit', '-i'], submitTemplate)
1550 if self.preserveUser:
1552 # Get last changelist number. Cannot easily get it from
1553 # the submit command output as the output is
1555 changelist = self.lastP4Changelist()
1556 self.modifyChangelistUser(changelist, p4User)
1558 # The rename/copy happened by applying a patch that created a
1559 # new file. This leaves it writable, which confuses p4.
1560 for f in pureRenameCopy:
1566 print "Submission cancelled, undoing p4 changes."
1567 for f in editedFiles:
1569 for f in filesToAdd:
1572 for f in filesToDelete:
1578 # Export git tags as p4 labels. Create a p4 label and then tag
1580 def exportGitTags(self, gitTags):
1581 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1582 if len(validLabelRegexp) == 0:
1583 validLabelRegexp = defaultLabelRegexp
1584 m = re.compile(validLabelRegexp)
1586 for name in gitTags:
1588 if not m.match(name):
1590 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1593 # Get the p4 commit this corresponds to
1594 logMessage = extractLogMessageFromGitCommit(name)
1595 values = extractSettingsGitLog(logMessage)
1597 if not values.has_key('change'):
1598 # a tag pointing to something not sent to p4; ignore
1600 print "git tag %s does not give a p4 commit" % name
1603 changelist = values['change']
1605 # Get the tag details.
1609 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1612 if re.match(r'tag\s+', l):
1614 elif re.match(r'\s*$', l):
1621 body = ["lightweight tag imported by git p4\n"]
1623 # Create the label - use the same view as the client spec we are using
1624 clientSpec = getClientSpec()
1626 labelTemplate = "Label: %s\n" % name
1627 labelTemplate += "Description:\n"
1629 labelTemplate += "\t" + b + "\n"
1630 labelTemplate += "View:\n"
1631 for depot_side in clientSpec.mappings:
1632 labelTemplate += "\t%s\n" % depot_side
1635 print "Would create p4 label %s for tag" % name
1636 elif self.prepare_p4_only:
1637 print "Not creating p4 label %s for tag due to option" \
1638 " --prepare-p4-only" % name
1640 p4_write_pipe(["label", "-i"], labelTemplate)
1643 p4_system(["tag", "-l", name] +
1644 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1647 print "created p4 label for tag %s" % name
1649 def run(self, args):
1651 self.master = currentGitBranch()
1652 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1653 die("Detecting current git branch failed!")
1654 elif len(args) == 1:
1655 self.master = args[0]
1656 if not branchExists(self.master):
1657 die("Branch %s does not exist" % self.master)
1661 allowSubmit = gitConfig("git-p4.allowSubmit")
1662 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1663 die("%s is not in git-p4.allowSubmit" % self.master)
1665 [upstream, settings] = findUpstreamBranchPoint()
1666 self.depotPath = settings['depot-paths'][0]
1667 if len(self.origin) == 0:
1668 self.origin = upstream
1670 if self.preserveUser:
1671 if not self.canChangeChangelists():
1672 die("Cannot preserve user names without p4 super-user or admin permissions")
1674 # if not set from the command line, try the config file
1675 if self.conflict_behavior is None:
1676 val = gitConfig("git-p4.conflict")
1678 if val not in self.conflict_behavior_choices:
1679 die("Invalid value '%s' for config git-p4.conflict" % val)
1682 self.conflict_behavior = val
1685 print "Origin branch is " + self.origin
1687 if len(self.depotPath) == 0:
1688 print "Internal error: cannot locate perforce depot path from existing branches"
1691 self.useClientSpec = False
1692 if gitConfigBool("git-p4.useclientspec"):
1693 self.useClientSpec = True
1694 if self.useClientSpec:
1695 self.clientSpecDirs = getClientSpec()
1697 # Check for the existance of P4 branches
1698 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1700 if self.useClientSpec and not branchesDetected:
1701 # all files are relative to the client spec
1702 self.clientPath = getClientRoot()
1704 self.clientPath = p4Where(self.depotPath)
1706 if self.clientPath == "":
1707 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1709 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1710 self.oldWorkingDirectory = os.getcwd()
1712 # ensure the clientPath exists
1713 new_client_dir = False
1714 if not os.path.exists(self.clientPath):
1715 new_client_dir = True
1716 os.makedirs(self.clientPath)
1718 chdir(self.clientPath, is_client_path=True)
1720 print "Would synchronize p4 checkout in %s" % self.clientPath
1722 print "Synchronizing p4 checkout..."
1724 # old one was destroyed, and maybe nobody told p4
1725 p4_sync("...", "-f")
1731 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1732 commits.append(line.strip())
1735 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1736 self.checkAuthorship = False
1738 self.checkAuthorship = True
1740 if self.preserveUser:
1741 self.checkValidP4Users(commits)
1744 # Build up a set of options to be passed to diff when
1745 # submitting each commit to p4.
1747 if self.detectRenames:
1748 # command-line -M arg
1749 self.diffOpts = "-M"
1751 # If not explicitly set check the config variable
1752 detectRenames = gitConfig("git-p4.detectRenames")
1754 if detectRenames.lower() == "false" or detectRenames == "":
1756 elif detectRenames.lower() == "true":
1757 self.diffOpts = "-M"
1759 self.diffOpts = "-M%s" % detectRenames
1761 # no command-line arg for -C or --find-copies-harder, just
1763 detectCopies = gitConfig("git-p4.detectCopies")
1764 if detectCopies.lower() == "false" or detectCopies == "":
1766 elif detectCopies.lower() == "true":
1767 self.diffOpts += " -C"
1769 self.diffOpts += " -C%s" % detectCopies
1771 if gitConfigBool("git-p4.detectCopiesHarder"):
1772 self.diffOpts += " --find-copies-harder"
1775 # Apply the commits, one at a time. On failure, ask if should
1776 # continue to try the rest of the patches, or quit.
1781 last = len(commits) - 1
1782 for i, commit in enumerate(commits):
1784 print " ", read_pipe(["git", "show", "-s",
1785 "--format=format:%h %s", commit])
1788 ok = self.applyCommit(commit)
1790 applied.append(commit)
1792 if self.prepare_p4_only and i < last:
1793 print "Processing only the first commit due to option" \
1794 " --prepare-p4-only"
1799 # prompt for what to do, or use the option/variable
1800 if self.conflict_behavior == "ask":
1801 print "What do you want to do?"
1802 response = raw_input("[s]kip this commit but apply"
1803 " the rest, or [q]uit? ")
1806 elif self.conflict_behavior == "skip":
1808 elif self.conflict_behavior == "quit":
1811 die("Unknown conflict_behavior '%s'" %
1812 self.conflict_behavior)
1814 if response[0] == "s":
1815 print "Skipping this commit, but applying the rest"
1817 if response[0] == "q":
1824 chdir(self.oldWorkingDirectory)
1828 elif self.prepare_p4_only:
1830 elif len(commits) == len(applied):
1831 print "All commits applied!"
1835 sync.branch = self.branch
1842 if len(applied) == 0:
1843 print "No commits applied."
1845 print "Applied only the commits marked with '*':"
1851 print star, read_pipe(["git", "show", "-s",
1852 "--format=format:%h %s", c])
1853 print "You will have to do 'git p4 sync' and rebase."
1855 if gitConfigBool("git-p4.exportLabels"):
1856 self.exportLabels = True
1858 if self.exportLabels:
1859 p4Labels = getP4Labels(self.depotPath)
1860 gitTags = getGitTags()
1862 missingGitTags = gitTags - p4Labels
1863 self.exportGitTags(missingGitTags)
1865 # exit with error unless everything applied perfectly
1866 if len(commits) != len(applied):
1872 """Represent a p4 view ("p4 help views"), and map files in a
1873 repo according to the view."""
1875 def __init__(self, client_name):
1877 self.client_prefix = "//%s/" % client_name
1878 # cache results of "p4 where" to lookup client file locations
1879 self.client_spec_path_cache = {}
1881 def append(self, view_line):
1882 """Parse a view line, splitting it into depot and client
1883 sides. Append to self.mappings, preserving order. This
1884 is only needed for tag creation."""
1886 # Split the view line into exactly two words. P4 enforces
1887 # structure on these lines that simplifies this quite a bit.
1889 # Either or both words may be double-quoted.
1890 # Single quotes do not matter.
1891 # Double-quote marks cannot occur inside the words.
1892 # A + or - prefix is also inside the quotes.
1893 # There are no quotes unless they contain a space.
1894 # The line is already white-space stripped.
1895 # The two words are separated by a single space.
1897 if view_line[0] == '"':
1898 # First word is double quoted. Find its end.
1899 close_quote_index = view_line.find('"', 1)
1900 if close_quote_index <= 0:
1901 die("No first-word closing quote found: %s" % view_line)
1902 depot_side = view_line[1:close_quote_index]
1903 # skip closing quote and space
1904 rhs_index = close_quote_index + 1 + 1
1906 space_index = view_line.find(" ")
1907 if space_index <= 0:
1908 die("No word-splitting space found: %s" % view_line)
1909 depot_side = view_line[0:space_index]
1910 rhs_index = space_index + 1
1912 # prefix + means overlay on previous mapping
1913 if depot_side.startswith("+"):
1914 depot_side = depot_side[1:]
1916 # prefix - means exclude this path, leave out of mappings
1918 if depot_side.startswith("-"):
1920 depot_side = depot_side[1:]
1923 self.mappings.append(depot_side)
1925 def convert_client_path(self, clientFile):
1926 # chop off //client/ part to make it relative
1927 if not clientFile.startswith(self.client_prefix):
1928 die("No prefix '%s' on clientFile '%s'" %
1929 (self.client_prefix, clientFile))
1930 return clientFile[len(self.client_prefix):]
1932 def update_client_spec_path_cache(self, files):
1933 """ Caching file paths by "p4 where" batch query """
1935 # List depot file paths exclude that already cached
1936 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1938 if len(fileArgs) == 0:
1939 return # All files in cache
1941 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1942 for res in where_result:
1943 if "code" in res and res["code"] == "error":
1944 # assume error is "... file(s) not in client view"
1946 if "clientFile" not in res:
1947 die("No clientFile in 'p4 where' output")
1949 # it will list all of them, but only one not unmap-ped
1951 if gitConfigBool("core.ignorecase"):
1952 res['depotFile'] = res['depotFile'].lower()
1953 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1955 # not found files or unmap files set to ""
1956 for depotFile in fileArgs:
1957 if gitConfigBool("core.ignorecase"):
1958 depotFile = depotFile.lower()
1959 if depotFile not in self.client_spec_path_cache:
1960 self.client_spec_path_cache[depotFile] = ""
1962 def map_in_client(self, depot_path):
1963 """Return the relative location in the client where this
1964 depot file should live. Returns "" if the file should
1965 not be mapped in the client."""
1967 if gitConfigBool("core.ignorecase"):
1968 depot_path = depot_path.lower()
1970 if depot_path in self.client_spec_path_cache:
1971 return self.client_spec_path_cache[depot_path]
1973 die( "Error: %s is not found in client spec path" % depot_path )
1976 class P4Sync(Command, P4UserMap):
1977 delete_actions = ( "delete", "move/delete", "purge" )
1980 Command.__init__(self)
1981 P4UserMap.__init__(self)
1983 optparse.make_option("--branch", dest="branch"),
1984 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1985 optparse.make_option("--changesfile", dest="changesFile"),
1986 optparse.make_option("--silent", dest="silent", action="store_true"),
1987 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1988 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1989 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1990 help="Import into refs/heads/ , not refs/remotes"),
1991 optparse.make_option("--max-changes", dest="maxChanges",
1992 help="Maximum number of changes to import"),
1993 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1994 help="Internal block size to use when iteratively calling p4 changes"),
1995 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1996 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1997 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1998 help="Only sync files that are included in the Perforce Client Spec"),
1999 optparse.make_option("-/", dest="cloneExclude",
2000 action="append", type="string",
2001 help="exclude depot path"),
2003 self.description = """Imports from Perforce into a git repository.\n
2005 //depot/my/project/ -- to import the current head
2006 //depot/my/project/@all -- to import everything
2007 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2009 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2011 self.usage += " //depot/path[@revRange]"
2013 self.createdBranches = set()
2014 self.committedChanges = set()
2016 self.detectBranches = False
2017 self.detectLabels = False
2018 self.importLabels = False
2019 self.changesFile = ""
2020 self.syncWithOrigin = True
2021 self.importIntoRemotes = True
2022 self.maxChanges = ""
2023 self.changes_block_size = None
2024 self.keepRepoPath = False
2025 self.depotPaths = None
2026 self.p4BranchesInGit = []
2027 self.cloneExclude = []
2028 self.useClientSpec = False
2029 self.useClientSpec_from_options = False
2030 self.clientSpecDirs = None
2031 self.tempBranches = []
2032 self.tempBranchLocation = "git-p4-tmp"
2034 if gitConfig("git-p4.syncFromOrigin") == "false":
2035 self.syncWithOrigin = False
2037 # This is required for the "append" cloneExclude action
2038 def ensure_value(self, attr, value):
2039 if not hasattr(self, attr) or getattr(self, attr) is None:
2040 setattr(self, attr, value)
2041 return getattr(self, attr)
2043 # Force a checkpoint in fast-import and wait for it to finish
2044 def checkpoint(self):
2045 self.gitStream.write("checkpoint\n\n")
2046 self.gitStream.write("progress checkpoint\n\n")
2047 out = self.gitOutput.readline()
2049 print "checkpoint finished: " + out
2051 def extractFilesFromCommit(self, commit):
2052 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2053 for path in self.cloneExclude]
2056 while commit.has_key("depotFile%s" % fnum):
2057 path = commit["depotFile%s" % fnum]
2059 if [p for p in self.cloneExclude
2060 if p4PathStartsWith(path, p)]:
2063 found = [p for p in self.depotPaths
2064 if p4PathStartsWith(path, p)]
2071 file["rev"] = commit["rev%s" % fnum]
2072 file["action"] = commit["action%s" % fnum]
2073 file["type"] = commit["type%s" % fnum]
2078 def stripRepoPath(self, path, prefixes):
2079 """When streaming files, this is called to map a p4 depot path
2080 to where it should go in git. The prefixes are either
2081 self.depotPaths, or self.branchPrefixes in the case of
2082 branch detection."""
2084 if self.useClientSpec:
2085 # branch detection moves files up a level (the branch name)
2086 # from what client spec interpretation gives
2087 path = self.clientSpecDirs.map_in_client(path)
2088 if self.detectBranches:
2089 for b in self.knownBranches:
2090 if path.startswith(b + "/"):
2091 path = path[len(b)+1:]
2093 elif self.keepRepoPath:
2094 # Preserve everything in relative path name except leading
2095 # //depot/; just look at first prefix as they all should
2096 # be in the same depot.
2097 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2098 if p4PathStartsWith(path, depot):
2099 path = path[len(depot):]
2103 if p4PathStartsWith(path, p):
2104 path = path[len(p):]
2107 path = wildcard_decode(path)
2110 def splitFilesIntoBranches(self, commit):
2111 """Look at each depotFile in the commit to figure out to what
2112 branch it belongs."""
2114 if self.clientSpecDirs:
2115 files = self.extractFilesFromCommit(commit)
2116 self.clientSpecDirs.update_client_spec_path_cache(files)
2120 while commit.has_key("depotFile%s" % fnum):
2121 path = commit["depotFile%s" % fnum]
2122 found = [p for p in self.depotPaths
2123 if p4PathStartsWith(path, p)]
2130 file["rev"] = commit["rev%s" % fnum]
2131 file["action"] = commit["action%s" % fnum]
2132 file["type"] = commit["type%s" % fnum]
2135 # start with the full relative path where this file would
2137 if self.useClientSpec:
2138 relPath = self.clientSpecDirs.map_in_client(path)
2140 relPath = self.stripRepoPath(path, self.depotPaths)
2142 for branch in self.knownBranches.keys():
2143 # add a trailing slash so that a commit into qt/4.2foo
2144 # doesn't end up in qt/4.2, e.g.
2145 if relPath.startswith(branch + "/"):
2146 if branch not in branches:
2147 branches[branch] = []
2148 branches[branch].append(file)
2153 # output one file from the P4 stream
2154 # - helper for streamP4Files
2156 def streamOneP4File(self, file, contents):
2157 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2159 sys.stderr.write("%s\n" % relPath)
2161 (type_base, type_mods) = split_p4_type(file["type"])
2164 if "x" in type_mods:
2166 if type_base == "symlink":
2168 # p4 print on a symlink sometimes contains "target\n";
2169 # if it does, remove the newline
2170 data = ''.join(contents)
2172 # Some version of p4 allowed creating a symlink that pointed
2173 # to nothing. This causes p4 errors when checking out such
2174 # a change, and errors here too. Work around it by ignoring
2175 # the bad symlink; hopefully a future change fixes it.
2176 print "\nIgnoring empty symlink in %s" % file['depotFile']
2178 elif data[-1] == '\n':
2179 contents = [data[:-1]]
2183 if type_base == "utf16":
2184 # p4 delivers different text in the python output to -G
2185 # than it does when using "print -o", or normal p4 client
2186 # operations. utf16 is converted to ascii or utf8, perhaps.
2187 # But ascii text saved as -t utf16 is completely mangled.
2188 # Invoke print -o to get the real contents.
2190 # On windows, the newlines will always be mangled by print, so put
2191 # them back too. This is not needed to the cygwin windows version,
2192 # just the native "NT" type.
2195 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2196 except Exception as e:
2197 if 'Translation of file content failed' in str(e):
2198 type_base = 'binary'
2202 if p4_version_string().find('/NT') >= 0:
2203 text = text.replace('\r\n', '\n')
2206 if type_base == "apple":
2207 # Apple filetype files will be streamed as a concatenation of
2208 # its appledouble header and the contents. This is useless
2209 # on both macs and non-macs. If using "print -q -o xx", it
2210 # will create "xx" with the data, and "%xx" with the header.
2211 # This is also not very useful.
2213 # Ideally, someday, this script can learn how to generate
2214 # appledouble files directly and import those to git, but
2215 # non-mac machines can never find a use for apple filetype.
2216 print "\nIgnoring apple filetype file %s" % file['depotFile']
2219 # Note that we do not try to de-mangle keywords on utf16 files,
2220 # even though in theory somebody may want that.
2221 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2223 regexp = re.compile(pattern, re.VERBOSE)
2224 text = ''.join(contents)
2225 text = regexp.sub(r'$\1$', text)
2229 relPath.decode('ascii')
2232 if gitConfig('git-p4.pathEncoding'):
2233 encoding = gitConfig('git-p4.pathEncoding')
2234 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2236 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2238 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2243 length = length + len(d)
2245 self.gitStream.write("data %d\n" % length)
2247 self.gitStream.write(d)
2248 self.gitStream.write("\n")
2250 def streamOneP4Deletion(self, file):
2251 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2253 sys.stderr.write("delete %s\n" % relPath)
2254 self.gitStream.write("D %s\n" % relPath)
2256 # handle another chunk of streaming data
2257 def streamP4FilesCb(self, marshalled):
2259 # catch p4 errors and complain
2261 if "code" in marshalled:
2262 if marshalled["code"] == "error":
2263 if "data" in marshalled:
2264 err = marshalled["data"].rstrip()
2267 if self.stream_have_file_info:
2268 if "depotFile" in self.stream_file:
2269 f = self.stream_file["depotFile"]
2270 # force a failure in fast-import, else an empty
2271 # commit will be made
2272 self.gitStream.write("\n")
2273 self.gitStream.write("die-now\n")
2274 self.gitStream.close()
2275 # ignore errors, but make sure it exits first
2276 self.importProcess.wait()
2278 die("Error from p4 print for %s: %s" % (f, err))
2280 die("Error from p4 print: %s" % err)
2282 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2283 # start of a new file - output the old one first
2284 self.streamOneP4File(self.stream_file, self.stream_contents)
2285 self.stream_file = {}
2286 self.stream_contents = []
2287 self.stream_have_file_info = False
2289 # pick up the new file information... for the
2290 # 'data' field we need to append to our array
2291 for k in marshalled.keys():
2293 self.stream_contents.append(marshalled['data'])
2295 self.stream_file[k] = marshalled[k]
2297 self.stream_have_file_info = True
2299 # Stream directly from "p4 files" into "git fast-import"
2300 def streamP4Files(self, files):
2306 # if using a client spec, only add the files that have
2307 # a path in the client
2308 if self.clientSpecDirs:
2309 if self.clientSpecDirs.map_in_client(f['path']) == "":
2312 filesForCommit.append(f)
2313 if f['action'] in self.delete_actions:
2314 filesToDelete.append(f)
2316 filesToRead.append(f)
2319 for f in filesToDelete:
2320 self.streamOneP4Deletion(f)
2322 if len(filesToRead) > 0:
2323 self.stream_file = {}
2324 self.stream_contents = []
2325 self.stream_have_file_info = False
2327 # curry self argument
2328 def streamP4FilesCbSelf(entry):
2329 self.streamP4FilesCb(entry)
2331 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2333 p4CmdList(["-x", "-", "print"],
2335 cb=streamP4FilesCbSelf)
2338 if self.stream_file.has_key('depotFile'):
2339 self.streamOneP4File(self.stream_file, self.stream_contents)
2341 def make_email(self, userid):
2342 if userid in self.users:
2343 return self.users[userid]
2345 return "%s <a@b>" % userid
2347 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2348 """ Stream a p4 tag.
2349 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2353 print "writing tag %s for commit %s" % (labelName, commit)
2354 gitStream.write("tag %s\n" % labelName)
2355 gitStream.write("from %s\n" % commit)
2357 if labelDetails.has_key('Owner'):
2358 owner = labelDetails["Owner"]
2362 # Try to use the owner of the p4 label, or failing that,
2363 # the current p4 user id.
2365 email = self.make_email(owner)
2367 email = self.make_email(self.p4UserId())
2368 tagger = "%s %s %s" % (email, epoch, self.tz)
2370 gitStream.write("tagger %s\n" % tagger)
2372 print "labelDetails=",labelDetails
2373 if labelDetails.has_key('Description'):
2374 description = labelDetails['Description']
2376 description = 'Label from git p4'
2378 gitStream.write("data %d\n" % len(description))
2379 gitStream.write(description)
2380 gitStream.write("\n")
2382 def commit(self, details, files, branch, parent = ""):
2383 epoch = details["time"]
2384 author = details["user"]
2387 print "commit into %s" % branch
2389 # start with reading files; if that fails, we should not
2393 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2394 new_files.append (f)
2396 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2398 if self.clientSpecDirs:
2399 self.clientSpecDirs.update_client_spec_path_cache(files)
2401 self.gitStream.write("commit %s\n" % branch)
2402 self.gitStream.write("mark :%s\n" % details["change"])
2403 self.committedChanges.add(int(details["change"]))
2405 if author not in self.users:
2406 self.getUserMapFromPerforceServer()
2407 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2409 self.gitStream.write("committer %s\n" % committer)
2411 self.gitStream.write("data <<EOT\n")
2412 self.gitStream.write(details["desc"])
2413 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2414 (','.join(self.branchPrefixes), details["change"]))
2415 if len(details['options']) > 0:
2416 self.gitStream.write(": options = %s" % details['options'])
2417 self.gitStream.write("]\nEOT\n\n")
2421 print "parent %s" % parent
2422 self.gitStream.write("from %s\n" % parent)
2424 self.streamP4Files(new_files)
2425 self.gitStream.write("\n")
2427 change = int(details["change"])
2429 if self.labels.has_key(change):
2430 label = self.labels[change]
2431 labelDetails = label[0]
2432 labelRevisions = label[1]
2434 print "Change %s is labelled %s" % (change, labelDetails)
2436 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2437 for p in self.branchPrefixes])
2439 if len(files) == len(labelRevisions):
2443 if info["action"] in self.delete_actions:
2445 cleanedFiles[info["depotFile"]] = info["rev"]
2447 if cleanedFiles == labelRevisions:
2448 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2452 print ("Tag %s does not match with change %s: files do not match."
2453 % (labelDetails["label"], change))
2457 print ("Tag %s does not match with change %s: file count is different."
2458 % (labelDetails["label"], change))
2460 # Build a dictionary of changelists and labels, for "detect-labels" option.
2461 def getLabels(self):
2464 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2465 if len(l) > 0 and not self.silent:
2466 print "Finding files belonging to labels in %s" % `self.depotPaths`
2469 label = output["label"]
2473 print "Querying files for label %s" % label
2474 for file in p4CmdList(["files"] +
2475 ["%s...@%s" % (p, label)
2476 for p in self.depotPaths]):
2477 revisions[file["depotFile"]] = file["rev"]
2478 change = int(file["change"])
2479 if change > newestChange:
2480 newestChange = change
2482 self.labels[newestChange] = [output, revisions]
2485 print "Label changes: %s" % self.labels.keys()
2487 # Import p4 labels as git tags. A direct mapping does not
2488 # exist, so assume that if all the files are at the same revision
2489 # then we can use that, or it's something more complicated we should
2491 def importP4Labels(self, stream, p4Labels):
2493 print "import p4 labels: " + ' '.join(p4Labels)
2495 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2496 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2497 if len(validLabelRegexp) == 0:
2498 validLabelRegexp = defaultLabelRegexp
2499 m = re.compile(validLabelRegexp)
2501 for name in p4Labels:
2504 if not m.match(name):
2506 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2509 if name in ignoredP4Labels:
2512 labelDetails = p4CmdList(['label', "-o", name])[0]
2514 # get the most recent changelist for each file in this label
2515 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2516 for p in self.depotPaths])
2518 if change.has_key('change'):
2519 # find the corresponding git commit; take the oldest commit
2520 changelist = int(change['change'])
2521 if changelist in self.committedChanges:
2522 gitCommit = ":%d" % changelist # use a fast-import mark
2525 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2526 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2527 if len(gitCommit) == 0:
2528 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2531 gitCommit = gitCommit.strip()
2534 # Convert from p4 time format
2536 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2538 print "Could not convert label time %s" % labelDetails['Update']
2541 when = int(time.mktime(tmwhen))
2542 self.streamTag(stream, name, labelDetails, gitCommit, when)
2544 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2547 print "Label %s has no changelists - possibly deleted?" % name
2550 # We can't import this label; don't try again as it will get very
2551 # expensive repeatedly fetching all the files for labels that will
2552 # never be imported. If the label is moved in the future, the
2553 # ignore will need to be removed manually.
2554 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2556 def guessProjectName(self):
2557 for p in self.depotPaths:
2560 p = p[p.strip().rfind("/") + 1:]
2561 if not p.endswith("/"):
2565 def getBranchMapping(self):
2566 lostAndFoundBranches = set()
2568 user = gitConfig("git-p4.branchUser")
2570 command = "branches -u %s" % user
2572 command = "branches"
2574 for info in p4CmdList(command):
2575 details = p4Cmd(["branch", "-o", info["branch"]])
2577 while details.has_key("View%s" % viewIdx):
2578 paths = details["View%s" % viewIdx].split(" ")
2579 viewIdx = viewIdx + 1
2580 # require standard //depot/foo/... //depot/bar/... mapping
2581 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2584 destination = paths[1]
2586 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2587 source = source[len(self.depotPaths[0]):-4]
2588 destination = destination[len(self.depotPaths[0]):-4]
2590 if destination in self.knownBranches:
2592 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2593 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2596 self.knownBranches[destination] = source
2598 lostAndFoundBranches.discard(destination)
2600 if source not in self.knownBranches:
2601 lostAndFoundBranches.add(source)
2603 # Perforce does not strictly require branches to be defined, so we also
2604 # check git config for a branch list.
2606 # Example of branch definition in git config file:
2608 # branchList=main:branchA
2609 # branchList=main:branchB
2610 # branchList=branchA:branchC
2611 configBranches = gitConfigList("git-p4.branchList")
2612 for branch in configBranches:
2614 (source, destination) = branch.split(":")
2615 self.knownBranches[destination] = source
2617 lostAndFoundBranches.discard(destination)
2619 if source not in self.knownBranches:
2620 lostAndFoundBranches.add(source)
2623 for branch in lostAndFoundBranches:
2624 self.knownBranches[branch] = branch
2626 def getBranchMappingFromGitBranches(self):
2627 branches = p4BranchesInGit(self.importIntoRemotes)
2628 for branch in branches.keys():
2629 if branch == "master":
2632 branch = branch[len(self.projectName):]
2633 self.knownBranches[branch] = branch
2635 def updateOptionDict(self, d):
2637 if self.keepRepoPath:
2638 option_keys['keepRepoPath'] = 1
2640 d["options"] = ' '.join(sorted(option_keys.keys()))
2642 def readOptions(self, d):
2643 self.keepRepoPath = (d.has_key('options')
2644 and ('keepRepoPath' in d['options']))
2646 def gitRefForBranch(self, branch):
2647 if branch == "main":
2648 return self.refPrefix + "master"
2650 if len(branch) <= 0:
2653 return self.refPrefix + self.projectName + branch
2655 def gitCommitByP4Change(self, ref, change):
2657 print "looking in ref " + ref + " for change %s using bisect..." % change
2660 latestCommit = parseRevision(ref)
2664 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2665 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2670 log = extractLogMessageFromGitCommit(next)
2671 settings = extractSettingsGitLog(log)
2672 currentChange = int(settings['change'])
2674 print "current change %s" % currentChange
2676 if currentChange == change:
2678 print "found %s" % next
2681 if currentChange < change:
2682 earliestCommit = "^%s" % next
2684 latestCommit = "%s" % next
2688 def importNewBranch(self, branch, maxChange):
2689 # make fast-import flush all changes to disk and update the refs using the checkpoint
2690 # command so that we can try to find the branch parent in the git history
2691 self.gitStream.write("checkpoint\n\n");
2692 self.gitStream.flush();
2693 branchPrefix = self.depotPaths[0] + branch + "/"
2694 range = "@1,%s" % maxChange
2695 #print "prefix" + branchPrefix
2696 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2697 if len(changes) <= 0:
2699 firstChange = changes[0]
2700 #print "first change in branch: %s" % firstChange
2701 sourceBranch = self.knownBranches[branch]
2702 sourceDepotPath = self.depotPaths[0] + sourceBranch
2703 sourceRef = self.gitRefForBranch(sourceBranch)
2704 #print "source " + sourceBranch
2706 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2707 #print "branch parent: %s" % branchParentChange
2708 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2709 if len(gitParent) > 0:
2710 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2711 #print "parent git commit: %s" % gitParent
2713 self.importChanges(changes)
2716 def searchParent(self, parent, branch, target):
2718 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2719 "--no-merges", parent]):
2721 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2724 print "Found parent of %s in commit %s" % (branch, blob)
2731 def importChanges(self, changes):
2733 for change in changes:
2734 description = p4_describe(change)
2735 self.updateOptionDict(description)
2738 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2743 if self.detectBranches:
2744 branches = self.splitFilesIntoBranches(description)
2745 for branch in branches.keys():
2747 branchPrefix = self.depotPaths[0] + branch + "/"
2748 self.branchPrefixes = [ branchPrefix ]
2752 filesForCommit = branches[branch]
2755 print "branch is %s" % branch
2757 self.updatedBranches.add(branch)
2759 if branch not in self.createdBranches:
2760 self.createdBranches.add(branch)
2761 parent = self.knownBranches[branch]
2762 if parent == branch:
2765 fullBranch = self.projectName + branch
2766 if fullBranch not in self.p4BranchesInGit:
2768 print("\n Importing new branch %s" % fullBranch);
2769 if self.importNewBranch(branch, change - 1):
2771 self.p4BranchesInGit.append(fullBranch)
2773 print("\n Resuming with change %s" % change);
2776 print "parent determined through known branches: %s" % parent
2778 branch = self.gitRefForBranch(branch)
2779 parent = self.gitRefForBranch(parent)
2782 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2784 if len(parent) == 0 and branch in self.initialParents:
2785 parent = self.initialParents[branch]
2786 del self.initialParents[branch]
2790 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2792 print "Creating temporary branch: " + tempBranch
2793 self.commit(description, filesForCommit, tempBranch)
2794 self.tempBranches.append(tempBranch)
2796 blob = self.searchParent(parent, branch, tempBranch)
2798 self.commit(description, filesForCommit, branch, blob)
2801 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2802 self.commit(description, filesForCommit, branch, parent)
2804 files = self.extractFilesFromCommit(description)
2805 self.commit(description, files, self.branch,
2807 # only needed once, to connect to the previous commit
2808 self.initialParent = ""
2810 print self.gitError.read()
2813 def importHeadRevision(self, revision):
2814 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2817 details["user"] = "git perforce import user"
2818 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2819 % (' '.join(self.depotPaths), revision))
2820 details["change"] = revision
2824 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2826 for info in p4CmdList(["files"] + fileArgs):
2828 if 'code' in info and info['code'] == 'error':
2829 sys.stderr.write("p4 returned an error: %s\n"
2831 if info['data'].find("must refer to client") >= 0:
2832 sys.stderr.write("This particular p4 error is misleading.\n")
2833 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2834 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2836 if 'p4ExitCode' in info:
2837 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2841 change = int(info["change"])
2842 if change > newestRevision:
2843 newestRevision = change
2845 if info["action"] in self.delete_actions:
2846 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2847 #fileCnt = fileCnt + 1
2850 for prop in ["depotFile", "rev", "action", "type" ]:
2851 details["%s%s" % (prop, fileCnt)] = info[prop]
2853 fileCnt = fileCnt + 1
2855 details["change"] = newestRevision
2857 # Use time from top-most change so that all git p4 clones of
2858 # the same p4 repo have the same commit SHA1s.
2859 res = p4_describe(newestRevision)
2860 details["time"] = res["time"]
2862 self.updateOptionDict(details)
2864 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2866 print "IO error with git fast-import. Is your git version recent enough?"
2867 print self.gitError.read()
2870 def run(self, args):
2871 self.depotPaths = []
2872 self.changeRange = ""
2873 self.previousDepotPaths = []
2874 self.hasOrigin = False
2876 # map from branch depot path to parent branch
2877 self.knownBranches = {}
2878 self.initialParents = {}
2880 if self.importIntoRemotes:
2881 self.refPrefix = "refs/remotes/p4/"
2883 self.refPrefix = "refs/heads/p4/"
2885 if self.syncWithOrigin:
2886 self.hasOrigin = originP4BranchesExist()
2889 print 'Syncing with origin first, using "git fetch origin"'
2890 system("git fetch origin")
2892 branch_arg_given = bool(self.branch)
2893 if len(self.branch) == 0:
2894 self.branch = self.refPrefix + "master"
2895 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2896 system("git update-ref %s refs/heads/p4" % self.branch)
2897 system("git branch -D p4")
2899 # accept either the command-line option, or the configuration variable
2900 if self.useClientSpec:
2901 # will use this after clone to set the variable
2902 self.useClientSpec_from_options = True
2904 if gitConfigBool("git-p4.useclientspec"):
2905 self.useClientSpec = True
2906 if self.useClientSpec:
2907 self.clientSpecDirs = getClientSpec()
2909 # TODO: should always look at previous commits,
2910 # merge with previous imports, if possible.
2913 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2915 # branches holds mapping from branch name to sha1
2916 branches = p4BranchesInGit(self.importIntoRemotes)
2918 # restrict to just this one, disabling detect-branches
2919 if branch_arg_given:
2920 short = self.branch.split("/")[-1]
2921 if short in branches:
2922 self.p4BranchesInGit = [ short ]
2924 self.p4BranchesInGit = branches.keys()
2926 if len(self.p4BranchesInGit) > 1:
2928 print "Importing from/into multiple branches"
2929 self.detectBranches = True
2930 for branch in branches.keys():
2931 self.initialParents[self.refPrefix + branch] = \
2935 print "branches: %s" % self.p4BranchesInGit
2938 for branch in self.p4BranchesInGit:
2939 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2941 settings = extractSettingsGitLog(logMsg)
2943 self.readOptions(settings)
2944 if (settings.has_key('depot-paths')
2945 and settings.has_key ('change')):
2946 change = int(settings['change']) + 1
2947 p4Change = max(p4Change, change)
2949 depotPaths = sorted(settings['depot-paths'])
2950 if self.previousDepotPaths == []:
2951 self.previousDepotPaths = depotPaths
2954 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2955 prev_list = prev.split("/")
2956 cur_list = cur.split("/")
2957 for i in range(0, min(len(cur_list), len(prev_list))):
2958 if cur_list[i] <> prev_list[i]:
2962 paths.append ("/".join(cur_list[:i + 1]))
2964 self.previousDepotPaths = paths
2967 self.depotPaths = sorted(self.previousDepotPaths)
2968 self.changeRange = "@%s,#head" % p4Change
2969 if not self.silent and not self.detectBranches:
2970 print "Performing incremental import into %s git branch" % self.branch
2972 # accept multiple ref name abbreviations:
2973 # refs/foo/bar/branch -> use it exactly
2974 # p4/branch -> prepend refs/remotes/ or refs/heads/
2975 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2976 if not self.branch.startswith("refs/"):
2977 if self.importIntoRemotes:
2978 prepend = "refs/remotes/"
2980 prepend = "refs/heads/"
2981 if not self.branch.startswith("p4/"):
2983 self.branch = prepend + self.branch
2985 if len(args) == 0 and self.depotPaths:
2987 print "Depot paths: %s" % ' '.join(self.depotPaths)
2989 if self.depotPaths and self.depotPaths != args:
2990 print ("previous import used depot path %s and now %s was specified. "
2991 "This doesn't work!" % (' '.join (self.depotPaths),
2995 self.depotPaths = sorted(args)
3000 # Make sure no revision specifiers are used when --changesfile
3002 bad_changesfile = False
3003 if len(self.changesFile) > 0:
3004 for p in self.depotPaths:
3005 if p.find("@") >= 0 or p.find("#") >= 0:
3006 bad_changesfile = True
3009 die("Option --changesfile is incompatible with revision specifiers")
3012 for p in self.depotPaths:
3013 if p.find("@") != -1:
3014 atIdx = p.index("@")
3015 self.changeRange = p[atIdx:]
3016 if self.changeRange == "@all":
3017 self.changeRange = ""
3018 elif ',' not in self.changeRange:
3019 revision = self.changeRange
3020 self.changeRange = ""
3022 elif p.find("#") != -1:
3023 hashIdx = p.index("#")
3024 revision = p[hashIdx:]
3026 elif self.previousDepotPaths == []:
3027 # pay attention to changesfile, if given, else import
3028 # the entire p4 tree at the head revision
3029 if len(self.changesFile) == 0:
3032 p = re.sub ("\.\.\.$", "", p)
3033 if not p.endswith("/"):
3038 self.depotPaths = newPaths
3040 # --detect-branches may change this for each branch
3041 self.branchPrefixes = self.depotPaths
3043 self.loadUserMapFromCache()
3045 if self.detectLabels:
3048 if self.detectBranches:
3049 ## FIXME - what's a P4 projectName ?
3050 self.projectName = self.guessProjectName()
3053 self.getBranchMappingFromGitBranches()
3055 self.getBranchMapping()
3057 print "p4-git branches: %s" % self.p4BranchesInGit
3058 print "initial parents: %s" % self.initialParents
3059 for b in self.p4BranchesInGit:
3063 b = b[len(self.projectName):]
3064 self.createdBranches.add(b)
3066 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3068 self.importProcess = subprocess.Popen(["git", "fast-import"],
3069 stdin=subprocess.PIPE,
3070 stdout=subprocess.PIPE,
3071 stderr=subprocess.PIPE);
3072 self.gitOutput = self.importProcess.stdout
3073 self.gitStream = self.importProcess.stdin
3074 self.gitError = self.importProcess.stderr
3077 self.importHeadRevision(revision)
3081 if len(self.changesFile) > 0:
3082 output = open(self.changesFile).readlines()
3085 changeSet.add(int(line))
3087 for change in changeSet:
3088 changes.append(change)
3092 # catch "git p4 sync" with no new branches, in a repo that
3093 # does not have any existing p4 branches
3095 if not self.p4BranchesInGit:
3096 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3098 # The default branch is master, unless --branch is used to
3099 # specify something else. Make sure it exists, or complain
3100 # nicely about how to use --branch.
3101 if not self.detectBranches:
3102 if not branch_exists(self.branch):
3103 if branch_arg_given:
3104 die("Error: branch %s does not exist." % self.branch)
3106 die("Error: no branch %s; perhaps specify one with --branch." %
3110 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3112 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3114 if len(self.maxChanges) > 0:
3115 changes = changes[:min(int(self.maxChanges), len(changes))]
3117 if len(changes) == 0:
3119 print "No changes to import!"
3121 if not self.silent and not self.detectBranches:
3122 print "Import destination: %s" % self.branch
3124 self.updatedBranches = set()
3126 if not self.detectBranches:
3128 # start a new branch
3129 self.initialParent = ""
3131 # build on a previous revision
3132 self.initialParent = parseRevision(self.branch)
3134 self.importChanges(changes)
3138 if len(self.updatedBranches) > 0:
3139 sys.stdout.write("Updated branches: ")
3140 for b in self.updatedBranches:
3141 sys.stdout.write("%s " % b)
3142 sys.stdout.write("\n")
3144 if gitConfigBool("git-p4.importLabels"):
3145 self.importLabels = True
3147 if self.importLabels:
3148 p4Labels = getP4Labels(self.depotPaths)
3149 gitTags = getGitTags()
3151 missingP4Labels = p4Labels - gitTags
3152 self.importP4Labels(self.gitStream, missingP4Labels)
3154 self.gitStream.close()
3155 if self.importProcess.wait() != 0:
3156 die("fast-import failed: %s" % self.gitError.read())
3157 self.gitOutput.close()
3158 self.gitError.close()
3160 # Cleanup temporary branches created during import
3161 if self.tempBranches != []:
3162 for branch in self.tempBranches:
3163 read_pipe("git update-ref -d %s" % branch)
3164 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3166 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3167 # a convenient shortcut refname "p4".
3168 if self.importIntoRemotes:
3169 head_ref = self.refPrefix + "HEAD"
3170 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3171 system(["git", "symbolic-ref", head_ref, self.branch])
3175 class P4Rebase(Command):
3177 Command.__init__(self)
3179 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3181 self.importLabels = False
3182 self.description = ("Fetches the latest revision from perforce and "
3183 + "rebases the current work (branch) against it")
3185 def run(self, args):
3187 sync.importLabels = self.importLabels
3190 return self.rebase()
3193 if os.system("git update-index --refresh") != 0:
3194 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3195 if len(read_pipe("git diff-index HEAD --")) > 0:
3196 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3198 [upstream, settings] = findUpstreamBranchPoint()
3199 if len(upstream) == 0:
3200 die("Cannot find upstream branchpoint for rebase")
3202 # the branchpoint may be p4/foo~3, so strip off the parent
3203 upstream = re.sub("~[0-9]+$", "", upstream)
3205 print "Rebasing the current branch onto %s" % upstream
3206 oldHead = read_pipe("git rev-parse HEAD").strip()
3207 system("git rebase %s" % upstream)
3208 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3211 class P4Clone(P4Sync):
3213 P4Sync.__init__(self)
3214 self.description = "Creates a new git repository and imports from Perforce into it"
3215 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3217 optparse.make_option("--destination", dest="cloneDestination",
3218 action='store', default=None,
3219 help="where to leave result of the clone"),
3220 optparse.make_option("--bare", dest="cloneBare",
3221 action="store_true", default=False),
3223 self.cloneDestination = None
3224 self.needsGit = False
3225 self.cloneBare = False
3227 def defaultDestination(self, args):
3228 ## TODO: use common prefix of args?
3230 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3231 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3232 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3233 depotDir = re.sub(r"/$", "", depotDir)
3234 return os.path.split(depotDir)[1]
3236 def run(self, args):
3240 if self.keepRepoPath and not self.cloneDestination:
3241 sys.stderr.write("Must specify destination for --keep-path\n")
3246 if not self.cloneDestination and len(depotPaths) > 1:
3247 self.cloneDestination = depotPaths[-1]
3248 depotPaths = depotPaths[:-1]
3250 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3251 for p in depotPaths:
3252 if not p.startswith("//"):
3253 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3256 if not self.cloneDestination:
3257 self.cloneDestination = self.defaultDestination(args)
3259 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3261 if not os.path.exists(self.cloneDestination):
3262 os.makedirs(self.cloneDestination)
3263 chdir(self.cloneDestination)
3265 init_cmd = [ "git", "init" ]
3267 init_cmd.append("--bare")
3268 retcode = subprocess.call(init_cmd)
3270 raise CalledProcessError(retcode, init_cmd)
3272 if not P4Sync.run(self, depotPaths):
3275 # create a master branch and check out a work tree
3276 if gitBranchExists(self.branch):
3277 system([ "git", "branch", "master", self.branch ])
3278 if not self.cloneBare:
3279 system([ "git", "checkout", "-f" ])
3281 print 'Not checking out any branch, use ' \
3282 '"git checkout -q -b master <branch>"'
3284 # auto-set this variable if invoked with --use-client-spec
3285 if self.useClientSpec_from_options:
3286 system("git config --bool git-p4.useclientspec true")
3290 class P4Branches(Command):
3292 Command.__init__(self)
3294 self.description = ("Shows the git branches that hold imports and their "
3295 + "corresponding perforce depot paths")
3296 self.verbose = False
3298 def run(self, args):
3299 if originP4BranchesExist():
3300 createOrUpdateBranchesFromOrigin()
3302 cmdline = "git rev-parse --symbolic "
3303 cmdline += " --remotes"
3305 for line in read_pipe_lines(cmdline):
3308 if not line.startswith('p4/') or line == "p4/HEAD":
3312 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3313 settings = extractSettingsGitLog(log)
3315 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3318 class HelpFormatter(optparse.IndentedHelpFormatter):
3320 optparse.IndentedHelpFormatter.__init__(self)
3322 def format_description(self, description):
3324 return description + "\n"
3328 def printUsage(commands):
3329 print "usage: %s <command> [options]" % sys.argv[0]
3331 print "valid commands: %s" % ", ".join(commands)
3333 print "Try %s <command> --help for command specific help." % sys.argv[0]
3338 "submit" : P4Submit,
3339 "commit" : P4Submit,
3341 "rebase" : P4Rebase,
3343 "rollback" : P4RollBack,
3344 "branches" : P4Branches
3349 if len(sys.argv[1:]) == 0:
3350 printUsage(commands.keys())
3353 cmdName = sys.argv[1]
3355 klass = commands[cmdName]
3358 print "unknown command %s" % cmdName
3360 printUsage(commands.keys())
3363 options = cmd.options
3364 cmd.gitdir = os.environ.get("GIT_DIR", None)
3368 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3370 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3372 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3374 description = cmd.description,
3375 formatter = HelpFormatter())
3377 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3379 verbose = cmd.verbose
3381 if cmd.gitdir == None:
3382 cmd.gitdir = os.path.abspath(".git")
3383 if not isValidGitDir(cmd.gitdir):
3384 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3385 if os.path.exists(cmd.gitdir):
3386 cdup = read_pipe("git rev-parse --show-cdup").strip()
3390 if not isValidGitDir(cmd.gitdir):
3391 if isValidGitDir(cmd.gitdir + "/.git"):
3392 cmd.gitdir += "/.git"
3394 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3396 os.environ["GIT_DIR"] = cmd.gitdir
3398 if not cmd.run(args):
3403 if __name__ == '__main__':