3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self, returncode, cmd):
39 self.returncode = returncode
42 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
52 def p4_build_cmd(cmd):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user = gitConfig("git-p4.user")
63 real_cmd += ["-u",user]
65 password = gitConfig("git-p4.password")
67 real_cmd += ["-P", password]
69 port = gitConfig("git-p4.port")
71 real_cmd += ["-p", port]
73 host = gitConfig("git-p4.host")
75 real_cmd += ["-H", host]
77 client = gitConfig("git-p4.client")
79 real_cmd += ["-c", client]
82 if isinstance(cmd,basestring):
83 real_cmd = ' '.join(real_cmd) + ' ' + cmd
88 def chdir(path, is_client_path=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
106 if not is_client_path:
108 os.environ['PWD'] = path
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform.system() == 'Windows':
113 free_bytes = ctypes.c_ulonglong(0)
114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115 return free_bytes.value
117 st = os.statvfs(os.getcwd())
118 return st.f_bavail * st.f_frsize
124 sys.stderr.write(msg + "\n")
127 def write_pipe(c, stdin):
129 sys.stderr.write('Writing pipe: %s\n' % str(c))
131 expand = isinstance(c,basestring)
132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
134 val = pipe.write(stdin)
137 die('Command failed: %s' % str(c))
141 def p4_write_pipe(c, stdin):
142 real_cmd = p4_build_cmd(c)
143 return write_pipe(real_cmd, stdin)
145 def read_pipe(c, ignore_error=False):
147 sys.stderr.write('Reading pipe: %s\n' % str(c))
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151 (out, err) = p.communicate()
152 if p.returncode != 0 and not ignore_error:
153 die('Command failed: %s\nError: %s' % (str(c), err))
156 def p4_read_pipe(c, ignore_error=False):
157 real_cmd = p4_build_cmd(c)
158 return read_pipe(real_cmd, ignore_error)
160 def read_pipe_lines(c):
162 sys.stderr.write('Reading pipe: %s\n' % str(c))
164 expand = isinstance(c, basestring)
165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
167 val = pipe.readlines()
168 if pipe.close() or p.wait():
169 die('Command failed: %s' % str(c))
173 def p4_read_pipe_lines(c):
174 """Specifically invoke p4 on the command supplied. """
175 real_cmd = p4_build_cmd(c)
176 return read_pipe_lines(real_cmd)
178 def p4_has_command(cmd):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd = p4_build_cmd(["help", cmd])
182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183 stderr=subprocess.PIPE)
185 return p.returncode == 0
187 def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
193 if not p4_has_command("move"):
195 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197 (out, err) = p.communicate()
198 # return code will be 1 in either case
199 if err.find("Invalid option") >= 0:
201 if err.find("disabled") >= 0:
203 # assume it failed because @... was invalid changelist
206 def system(cmd, ignore_error=False):
207 expand = isinstance(cmd,basestring)
209 sys.stderr.write("executing %s\n" % str(cmd))
210 retcode = subprocess.call(cmd, shell=expand)
211 if retcode and not ignore_error:
212 raise CalledProcessError(retcode, cmd)
217 """Specifically invoke p4 as the system command. """
218 real_cmd = p4_build_cmd(cmd)
219 expand = isinstance(real_cmd, basestring)
220 retcode = subprocess.call(real_cmd, shell=expand)
222 raise CalledProcessError(retcode, real_cmd)
224 _p4_version_string = None
225 def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
234 global _p4_version_string
235 if not _p4_version_string:
236 a = p4_read_pipe_lines(["-V"])
237 _p4_version_string = a[-1].rstrip()
238 return _p4_version_string
240 def p4_integrate(src, dest):
241 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
243 def p4_sync(f, *options):
244 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
247 # forcibly add file names with wildcards
248 if wildcard_present(f):
249 p4_system(["add", "-f", f])
251 p4_system(["add", f])
254 p4_system(["delete", wildcard_encode(f)])
256 def p4_edit(f, *options):
257 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
260 p4_system(["revert", wildcard_encode(f)])
262 def p4_reopen(type, f):
263 p4_system(["reopen", "-t", type, wildcard_encode(f)])
265 def p4_move(src, dest):
266 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
268 def p4_last_change():
269 results = p4CmdList(["changes", "-m", "1"])
270 return int(results[0]['change'])
272 def p4_describe(change):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
277 ds = p4CmdList(["describe", "-s", str(change)])
279 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
283 if "p4ExitCode" in d:
284 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
287 if d["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
291 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers. See "p4 help filetypes"
298 # for a list and explanation.
300 def split_p4_type(p4type):
302 p4_filetypes_historical = {
303 "ctempobj": "binary+Sw",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
315 "xtempobj": "binary+Swx",
317 "xunicode": "unicode+x",
320 if p4type in p4_filetypes_historical:
321 p4type = p4_filetypes_historical[p4type]
323 s = p4type.split("+")
331 # return the raw p4 type of a file (text, text+ko, etc)
334 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
335 return results[0]['headType']
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
341 def p4_keywords_regexp_for_type(base, type_mods):
342 if base in ("text", "unicode", "binary"):
344 if "ko" in type_mods:
346 elif "k" in type_mods:
347 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
353 (:[^$\n]+)? # possibly an old expansion, followed by...
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
365 def p4_keywords_regexp_for_file(file):
366 if not os.path.exists(file):
369 (type_base, type_mods) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base, type_mods)
372 def setP4ExecBit(file, mode):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
378 if not isModeExec(mode):
379 p4Type = getP4OpenedType(file)
380 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382 if p4Type[-1] == "+":
383 p4Type = p4Type[0:-1]
385 p4_reopen(p4Type, file)
387 def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
390 result = p4_read_pipe(["opened", wildcard_encode(file)])
391 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
393 return match.group(1)
395 die("Could not determine file type for %s (result: '%s')" % (file, result))
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths):
400 if isinstance(depotPaths,basestring):
401 depotPaths = [depotPaths]
403 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
409 # Return the set of all git tags
412 for line in read_pipe_lines(["git", "tag"]):
417 def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
424 def parseDiffTreeEntry(entry):
425 """Parses a single diff tree entry into its component elements.
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
441 If the pattern is not matched, None is returned."""
443 match = diffTreePattern().next().match(entry)
446 'src_mode': match.group(1),
447 'dst_mode': match.group(2),
448 'src_sha1': match.group(3),
449 'dst_sha1': match.group(4),
450 'status': match.group(5),
451 'status_score': match.group(6),
452 'src': match.group(7),
453 'dst': match.group(10)
457 def isModeExec(mode):
458 # Returns True if the given git mode represents an executable file,
460 return mode[-3:] == "755"
462 def isModeExecChanged(src_mode, dst_mode):
463 return isModeExec(src_mode) != isModeExec(dst_mode)
465 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
467 if isinstance(cmd,basestring):
474 cmd = p4_build_cmd(cmd)
476 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
482 if stdin is not None:
483 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
484 if isinstance(stdin,basestring):
485 stdin_file.write(stdin)
488 stdin_file.write(i + '\n')
492 p4 = subprocess.Popen(cmd,
495 stdout=subprocess.PIPE)
500 entry = marshal.load(p4.stdout)
510 entry["p4ExitCode"] = exitCode
516 list = p4CmdList(cmd)
522 def p4Where(depotPath):
523 if not depotPath.endswith("/"):
525 depotPathLong = depotPath + "..."
526 outputList = p4CmdList(["where", depotPathLong])
528 for entry in outputList:
529 if "depotFile" in entry:
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
535 elif "data" in entry:
536 data = entry.get("data")
537 space = data.find(" ")
538 if data[:space] == depotPath:
543 if output["code"] == "error":
547 clientPath = output.get("path")
548 elif "data" in output:
549 data = output.get("data")
550 lastSpace = data.rfind(" ")
551 clientPath = data[lastSpace + 1:]
553 if clientPath.endswith("..."):
554 clientPath = clientPath[:-3]
557 def currentGitBranch():
558 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
565 def isValidGitDir(path):
566 if (os.path.exists(path + "/HEAD")
567 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
571 def parseRevision(ref):
572 return read_pipe("git rev-parse %s" % ref).strip()
574 def branchExists(ref):
575 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
579 def extractLogMessageFromGitCommit(commit):
582 ## fixme: title is first line of commit, not 1st paragraph.
584 for log in read_pipe_lines("git cat-file commit %s" % commit):
593 def extractSettingsGitLog(log):
595 for line in log.split("\n"):
597 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
601 assignments = m.group(1).split (':')
602 for a in assignments:
604 key = vals[0].strip()
605 val = ('='.join (vals[1:])).strip()
606 if val.endswith ('\"') and val.startswith('"'):
611 paths = values.get("depot-paths")
613 paths = values.get("depot-path")
615 values['depot-paths'] = paths.split(',')
618 def gitBranchExists(branch):
619 proc = subprocess.Popen(["git", "rev-parse", branch],
620 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
621 return proc.wait() == 0;
625 def gitConfig(key, typeSpecifier=None):
626 if not _gitConfig.has_key(key):
627 cmd = [ "git", "config" ]
629 cmd += [ typeSpecifier ]
631 s = read_pipe(cmd, ignore_error=True)
632 _gitConfig[key] = s.strip()
633 return _gitConfig[key]
635 def gitConfigBool(key):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
640 if not _gitConfig.has_key(key):
641 _gitConfig[key] = gitConfig(key, '--bool') == "true"
642 return _gitConfig[key]
644 def gitConfigInt(key):
645 if not _gitConfig.has_key(key):
646 cmd = [ "git", "config", "--int", key ]
647 s = read_pipe(cmd, ignore_error=True)
650 _gitConfig[key] = int(gitConfig(key, '--int'))
652 _gitConfig[key] = None
653 return _gitConfig[key]
655 def gitConfigList(key):
656 if not _gitConfig.has_key(key):
657 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658 _gitConfig[key] = s.strip().split(os.linesep)
659 if _gitConfig[key] == ['']:
661 return _gitConfig[key]
663 def p4BranchesInGit(branchesAreInRemotes=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
672 cmdline = "git rev-parse --symbolic "
673 if branchesAreInRemotes:
674 cmdline += "--remotes"
676 cmdline += "--branches"
678 for line in read_pipe_lines(cmdline):
682 if not line.startswith('p4/'):
684 # special symbolic ref to p4/master
685 if line == "p4/HEAD":
688 # strip off p4/ prefix
689 branch = line[len("p4/"):]
691 branches[branch] = parseRevision(line)
695 def branch_exists(branch):
696 """Make sure that the given ref name really exists."""
698 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700 out, _ = p.communicate()
703 # expect exactly one line of output: the branch name
704 return out.rstrip() == branch
706 def findUpstreamBranchPoint(head = "HEAD"):
707 branches = p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath = {}
710 for branch in branches.keys():
711 tip = branches[branch]
712 log = extractLogMessageFromGitCommit(tip)
713 settings = extractSettingsGitLog(log)
714 if settings.has_key("depot-paths"):
715 paths = ",".join(settings["depot-paths"])
716 branchByDepotPath[paths] = "remotes/p4/" + branch
720 while parent < 65535:
721 commit = head + "~%s" % parent
722 log = extractLogMessageFromGitCommit(commit)
723 settings = extractSettingsGitLog(log)
724 if settings.has_key("depot-paths"):
725 paths = ",".join(settings["depot-paths"])
726 if branchByDepotPath.has_key(paths):
727 return [branchByDepotPath[paths], settings]
731 return ["", settings]
733 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
738 originPrefix = "origin/p4/"
740 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
742 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
745 headName = line[len(originPrefix):]
746 remoteHead = localRefPrefix + headName
749 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750 if (not original.has_key('depot-paths')
751 or not original.has_key('change')):
755 if not gitBranchExists(remoteHead):
757 print "creating %s" % remoteHead
760 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761 if settings.has_key('change') > 0:
762 if settings['depot-paths'] == original['depot-paths']:
763 originP4Change = int(original['change'])
764 p4Change = int(settings['change'])
765 if originP4Change > p4Change:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead, originP4Change,
769 remoteHead, p4Change))
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead, ','.join(original['depot-paths']),
775 remoteHead, ','.join(settings['depot-paths'])))
778 system("git update-ref %s %s" % (remoteHead, originHead))
780 def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
784 def p4ParseNumericChangeRange(parts):
785 changeStart = int(parts[0][1:])
786 if parts[1] == '#head':
787 changeEnd = p4_last_change()
789 changeEnd = int(parts[1])
791 return (changeStart, changeEnd)
793 def chooseBlockSize(blockSize):
797 return defaultBlockSize
799 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
808 if changeRange is None or changeRange == '':
810 changeEnd = p4_last_change()
811 block_size = chooseBlockSize(requestedBlockSize)
813 parts = changeRange.split(',')
814 assert len(parts) == 2
816 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817 block_size = chooseBlockSize(requestedBlockSize)
819 changeStart = parts[0][1:]
821 if requestedBlockSize:
822 die("cannot use --changes-block-size with non-numeric revisions")
827 # Retrieve changes a block at a time, to prevent running
828 # into a MaxResults/MaxScanRows error from the server.
834 end = min(changeEnd, changeStart + block_size)
835 revisionRange = "%d,%d" % (changeStart, end)
837 revisionRange = "%s,%s" % (changeStart, changeEnd)
840 cmd += ["%s...@%s" % (p, revisionRange)]
842 # Insert changes in chronological order
843 for line in reversed(p4_read_pipe_lines(cmd)):
844 changes.append(int(line.split(" ")[1]))
852 changeStart = end + 1
854 changes = sorted(changes)
857 def p4PathStartsWith(path, prefix):
858 # This method tries to remedy a potential mixed-case issue:
860 # If UserA adds //depot/DirA/file1
861 # and UserB adds //depot/dira/file2
863 # we may or may not have a problem. If you have core.ignorecase=true,
864 # we treat DirA and dira as the same directory
865 if gitConfigBool("core.ignorecase"):
866 return path.lower().startswith(prefix.lower())
867 return path.startswith(prefix)
870 """Look at the p4 client spec, create a View() object that contains
871 all the mappings, and return it."""
873 specList = p4CmdList("client -o")
874 if len(specList) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' %
878 # dictionary of all client parameters
882 client_name = entry["Client"]
884 # just the keys that start with "View"
885 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
888 view = View(client_name)
890 # append the lines, in order, to the view
891 for view_num in range(len(view_keys)):
892 k = "View%d" % view_num
893 if k not in view_keys:
894 die("Expected view key %s missing" % k)
895 view.append(entry[k])
900 """Grab the client directory."""
902 output = p4CmdList("client -o")
904 die('Output from "client -o" is %d lines, expecting 1' % len(output))
907 if "Root" not in entry:
908 die('Client has no "Root"')
913 # P4 wildcards are not allowed in filenames. P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
917 def wildcard_decode(path):
918 # Search for and fix just these four characters. Do % last so
919 # that fixing it does not inadvertently create new %-escapes.
920 # Cannot have * in a filename in windows; untested as to
921 # what p4 would do in such a case.
922 if not platform.system() == "Windows":
923 path = path.replace("%2A", "*")
924 path = path.replace("%23", "#") \
925 .replace("%40", "@") \
929 def wildcard_encode(path):
930 # do % first to avoid double-encoding the %s introduced here
931 path = path.replace("%", "%25") \
932 .replace("*", "%2A") \
933 .replace("#", "%23") \
937 def wildcard_present(path):
938 m = re.search("[*#@%]", path)
941 class LargeFileSystem(object):
942 """Base class for large file system support."""
944 def __init__(self, writeToGitStream):
945 self.largeFiles = set()
946 self.writeToGitStream = writeToGitStream
948 def generatePointer(self, cloneDestination, contentFile):
949 """Return the content of a pointer file that is stored in Git instead of
950 the actual content."""
951 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
953 def pushFile(self, localLargeFile):
954 """Push the actual content which is not stored in the Git repository to
956 assert False, "Method 'pushFile' required in " + self.__class__.__name__
958 def hasLargeFileExtension(self, relPath):
961 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
965 def generateTempFile(self, contents):
966 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
970 return contentFile.name
972 def exceedsLargeFileThreshold(self, relPath, contents):
973 if gitConfigInt('git-p4.largeFileThreshold'):
974 contentsSize = sum(len(d) for d in contents)
975 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
977 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978 contentsSize = sum(len(d) for d in contents)
979 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
981 contentTempFile = self.generateTempFile(contents)
982 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
986 compressedContentsSize = zf.infolist()[0].compress_size
987 os.remove(contentTempFile)
988 os.remove(compressedContentFile.name)
989 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
993 def addLargeFile(self, relPath):
994 self.largeFiles.add(relPath)
996 def removeLargeFile(self, relPath):
997 self.largeFiles.remove(relPath)
999 def isLargeFile(self, relPath):
1000 return relPath in self.largeFiles
1002 def processContent(self, git_mode, relPath, contents):
1003 """Processes the content of git fast import. This method decides if a
1004 file is stored in the large file system and handles all necessary
1006 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007 contentTempFile = self.generateTempFile(contents)
1008 (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1010 # Move temp file to final location in large file system
1011 largeFileDir = os.path.dirname(localLargeFile)
1012 if not os.path.isdir(largeFileDir):
1013 os.makedirs(largeFileDir)
1014 shutil.move(contentTempFile, localLargeFile)
1015 self.addLargeFile(relPath)
1016 if gitConfigBool('git-p4.largeFilePush'):
1017 self.pushFile(localLargeFile)
1019 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020 return (git_mode, contents)
1022 class MockLFS(LargeFileSystem):
1023 """Mock large file system for testing."""
1025 def generatePointer(self, contentFile):
1026 """The pointer content is the original content prefixed with "pointer-".
1027 The local filename of the large file storage is derived from the file content.
1029 with open(contentFile, 'r') as f:
1032 pointerContents = 'pointer-' + content
1033 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034 return (gitMode, pointerContents, localLargeFile)
1036 def pushFile(self, localLargeFile):
1037 """The remote filename of the large file storage is the same as the local
1038 one but in a different directory.
1040 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041 if not os.path.exists(remotePath):
1042 os.makedirs(remotePath)
1043 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1045 class GitLFS(LargeFileSystem):
1046 """Git LFS as backend for the git-p4 large file system.
1047 See https://git-lfs.github.com/ for details."""
1049 def __init__(self, *args):
1050 LargeFileSystem.__init__(self, *args)
1051 self.baseGitAttributes = []
1053 def generatePointer(self, contentFile):
1054 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055 mode and content which is stored in the Git repository instead of
1056 the actual content. Return also the new location of the actual
1059 pointerProcess = subprocess.Popen(
1060 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061 stdout=subprocess.PIPE
1063 pointerFile = pointerProcess.stdout.read()
1064 if pointerProcess.wait():
1065 os.remove(contentFile)
1066 die('git-lfs pointer command failed. Did you install the extension?')
1068 # Git LFS removed the preamble in the output of the 'pointer' command
1069 # starting from version 1.2.0. Check for the preamble here to support
1071 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1072 if pointerFile.startswith('Git LFS pointer for'):
1073 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1075 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1076 localLargeFile = os.path.join(
1078 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1081 # LFS Spec states that pointer files should not have the executable bit set.
1083 return (gitMode, pointerFile, localLargeFile)
1085 def pushFile(self, localLargeFile):
1086 uploadProcess = subprocess.Popen(
1087 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1089 if uploadProcess.wait():
1090 die('git-lfs push command failed. Did you define a remote?')
1092 def generateGitAttributes(self):
1094 self.baseGitAttributes +
1098 '# Git LFS (see https://git-lfs.github.com/)\n',
1101 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1102 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1104 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1105 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1109 def addLargeFile(self, relPath):
1110 LargeFileSystem.addLargeFile(self, relPath)
1111 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1113 def removeLargeFile(self, relPath):
1114 LargeFileSystem.removeLargeFile(self, relPath)
1115 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1117 def processContent(self, git_mode, relPath, contents):
1118 if relPath == '.gitattributes':
1119 self.baseGitAttributes = contents
1120 return (git_mode, self.generateGitAttributes())
1122 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1126 self.usage = "usage: %prog [options]"
1127 self.needsGit = True
1128 self.verbose = False
1132 self.userMapFromPerforceServer = False
1133 self.myP4UserId = None
1137 return self.myP4UserId
1139 results = p4CmdList("user -o")
1141 if r.has_key('User'):
1142 self.myP4UserId = r['User']
1144 die("Could not find your p4 user id")
1146 def p4UserIsMe(self, p4User):
1147 # return True if the given p4 user is actually me
1148 me = self.p4UserId()
1149 if not p4User or p4User != me:
1154 def getUserCacheFilename(self):
1155 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1156 return home + "/.gitp4-usercache.txt"
1158 def getUserMapFromPerforceServer(self):
1159 if self.userMapFromPerforceServer:
1164 for output in p4CmdList("users"):
1165 if not output.has_key("User"):
1167 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1168 self.emails[output["Email"]] = output["User"]
1170 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1171 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1172 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1173 if mapUser and len(mapUser[0]) == 3:
1174 user = mapUser[0][0]
1175 fullname = mapUser[0][1]
1176 email = mapUser[0][2]
1177 self.users[user] = fullname + " <" + email + ">"
1178 self.emails[email] = user
1181 for (key, val) in self.users.items():
1182 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1184 open(self.getUserCacheFilename(), "wb").write(s)
1185 self.userMapFromPerforceServer = True
1187 def loadUserMapFromCache(self):
1189 self.userMapFromPerforceServer = False
1191 cache = open(self.getUserCacheFilename(), "rb")
1192 lines = cache.readlines()
1195 entry = line.strip().split("\t")
1196 self.users[entry[0]] = entry[1]
1198 self.getUserMapFromPerforceServer()
1200 class P4Debug(Command):
1202 Command.__init__(self)
1204 self.description = "A tool to debug the output of p4 -G."
1205 self.needsGit = False
1207 def run(self, args):
1209 for output in p4CmdList(args):
1210 print 'Element: %d' % j
1215 class P4RollBack(Command):
1217 Command.__init__(self)
1219 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1221 self.description = "A tool to debug the multi-branch import. Don't use :)"
1222 self.rollbackLocalBranches = False
1224 def run(self, args):
1227 maxChange = int(args[0])
1229 if "p4ExitCode" in p4Cmd("changes -m 1"):
1230 die("Problems executing p4");
1232 if self.rollbackLocalBranches:
1233 refPrefix = "refs/heads/"
1234 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1236 refPrefix = "refs/remotes/"
1237 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1240 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1242 ref = refPrefix + line
1243 log = extractLogMessageFromGitCommit(ref)
1244 settings = extractSettingsGitLog(log)
1246 depotPaths = settings['depot-paths']
1247 change = settings['change']
1251 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1252 for p in depotPaths]))) == 0:
1253 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1254 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1257 while change and int(change) > maxChange:
1260 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1261 system("git update-ref %s \"%s^\"" % (ref, ref))
1262 log = extractLogMessageFromGitCommit(ref)
1263 settings = extractSettingsGitLog(log)
1266 depotPaths = settings['depot-paths']
1267 change = settings['change']
1270 print "%s rewound to %s" % (ref, change)
1274 class P4Submit(Command, P4UserMap):
1276 conflict_behavior_choices = ("ask", "skip", "quit")
1279 Command.__init__(self)
1280 P4UserMap.__init__(self)
1282 optparse.make_option("--origin", dest="origin"),
1283 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1284 # preserve the user, requires relevant p4 permissions
1285 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1286 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1287 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1288 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1289 optparse.make_option("--conflict", dest="conflict_behavior",
1290 choices=self.conflict_behavior_choices),
1291 optparse.make_option("--branch", dest="branch"),
1292 optparse.make_option("--shelve", dest="shelve", action="store_true",
1293 help="Shelve instead of submit. Shelved files are reverted, "
1294 "restoring the workspace to the state before the shelve"),
1296 self.description = "Submit changes from git to the perforce depot."
1297 self.usage += " [name of git branch to submit into perforce depot]"
1299 self.detectRenames = False
1300 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1301 self.dry_run = False
1303 self.prepare_p4_only = False
1304 self.conflict_behavior = None
1305 self.isWindows = (platform.system() == "Windows")
1306 self.exportLabels = False
1307 self.p4HasMoveCommand = p4_has_move_command()
1310 if gitConfig('git-p4.largeFileSystem'):
1311 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1314 if len(p4CmdList("opened ...")) > 0:
1315 die("You have files opened with perforce! Close them before starting the sync.")
1317 def separate_jobs_from_description(self, message):
1318 """Extract and return a possible Jobs field in the commit
1319 message. It goes into a separate section in the p4 change
1322 A jobs line starts with "Jobs:" and looks like a new field
1323 in a form. Values are white-space separated on the same
1324 line or on following lines that start with a tab.
1326 This does not parse and extract the full git commit message
1327 like a p4 form. It just sees the Jobs: line as a marker
1328 to pass everything from then on directly into the p4 form,
1329 but outside the description section.
1331 Return a tuple (stripped log message, jobs string)."""
1333 m = re.search(r'^Jobs:', message, re.MULTILINE)
1335 return (message, None)
1337 jobtext = message[m.start():]
1338 stripped_message = message[:m.start()].rstrip()
1339 return (stripped_message, jobtext)
1341 def prepareLogMessage(self, template, message, jobs):
1342 """Edits the template returned from "p4 change -o" to insert
1343 the message in the Description field, and the jobs text in
1347 inDescriptionSection = False
1349 for line in template.split("\n"):
1350 if line.startswith("#"):
1351 result += line + "\n"
1354 if inDescriptionSection:
1355 if line.startswith("Files:") or line.startswith("Jobs:"):
1356 inDescriptionSection = False
1357 # insert Jobs section
1359 result += jobs + "\n"
1363 if line.startswith("Description:"):
1364 inDescriptionSection = True
1366 for messageLine in message.split("\n"):
1367 line += "\t" + messageLine + "\n"
1369 result += line + "\n"
1373 def patchRCSKeywords(self, file, pattern):
1374 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1375 (handle, outFileName) = tempfile.mkstemp(dir='.')
1377 outFile = os.fdopen(handle, "w+")
1378 inFile = open(file, "r")
1379 regexp = re.compile(pattern, re.VERBOSE)
1380 for line in inFile.readlines():
1381 line = regexp.sub(r'$\1$', line)
1385 # Forcibly overwrite the original file
1387 shutil.move(outFileName, file)
1389 # cleanup our temporary file
1390 os.unlink(outFileName)
1391 print "Failed to strip RCS keywords in %s" % file
1394 print "Patched up RCS keywords in %s" % file
1396 def p4UserForCommit(self,id):
1397 # Return the tuple (perforce user,git email) for a given git commit id
1398 self.getUserMapFromPerforceServer()
1399 gitEmail = read_pipe(["git", "log", "--max-count=1",
1400 "--format=%ae", id])
1401 gitEmail = gitEmail.strip()
1402 if not self.emails.has_key(gitEmail):
1403 return (None,gitEmail)
1405 return (self.emails[gitEmail],gitEmail)
1407 def checkValidP4Users(self,commits):
1408 # check if any git authors cannot be mapped to p4 users
1410 (user,email) = self.p4UserForCommit(id)
1412 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1413 if gitConfigBool("git-p4.allowMissingP4Users"):
1416 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1418 def lastP4Changelist(self):
1419 # Get back the last changelist number submitted in this client spec. This
1420 # then gets used to patch up the username in the change. If the same
1421 # client spec is being used by multiple processes then this might go
1423 results = p4CmdList("client -o") # find the current client
1426 if r.has_key('Client'):
1427 client = r['Client']
1430 die("could not get client spec")
1431 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1433 if r.has_key('change'):
1435 die("Could not get changelist number for last submit - cannot patch up user details")
1437 def modifyChangelistUser(self, changelist, newUser):
1438 # fixup the user field of a changelist after it has been submitted.
1439 changes = p4CmdList("change -o %s" % changelist)
1440 if len(changes) != 1:
1441 die("Bad output from p4 change modifying %s to user %s" %
1442 (changelist, newUser))
1445 if c['User'] == newUser: return # nothing to do
1447 input = marshal.dumps(c)
1449 result = p4CmdList("change -f -i", stdin=input)
1451 if r.has_key('code'):
1452 if r['code'] == 'error':
1453 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1454 if r.has_key('data'):
1455 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1457 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1459 def canChangeChangelists(self):
1460 # check to see if we have p4 admin or super-user permissions, either of
1461 # which are required to modify changelists.
1462 results = p4CmdList(["protects", self.depotPath])
1464 if r.has_key('perm'):
1465 if r['perm'] == 'admin':
1467 if r['perm'] == 'super':
1471 def prepareSubmitTemplate(self):
1472 """Run "p4 change -o" to grab a change specification template.
1473 This does not use "p4 -G", as it is nice to keep the submission
1474 template in original order, since a human might edit it.
1476 Remove lines in the Files section that show changes to files
1477 outside the depot path we're committing into."""
1479 [upstream, settings] = findUpstreamBranchPoint()
1482 inFilesSection = False
1483 for line in p4_read_pipe_lines(['change', '-o']):
1484 if line.endswith("\r\n"):
1485 line = line[:-2] + "\n"
1487 if line.startswith("\t"):
1488 # path starts and ends with a tab
1490 lastTab = path.rfind("\t")
1492 path = path[:lastTab]
1493 if settings.has_key('depot-paths'):
1494 if not [p for p in settings['depot-paths']
1495 if p4PathStartsWith(path, p)]:
1498 if not p4PathStartsWith(path, self.depotPath):
1501 inFilesSection = False
1503 if line.startswith("Files:"):
1504 inFilesSection = True
1510 def edit_template(self, template_file):
1511 """Invoke the editor to let the user change the submission
1512 message. Return true if okay to continue with the submit."""
1514 # if configured to skip the editing part, just submit
1515 if gitConfigBool("git-p4.skipSubmitEdit"):
1518 # look at the modification time, to check later if the user saved
1520 mtime = os.stat(template_file).st_mtime
1523 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1524 editor = os.environ.get("P4EDITOR")
1526 editor = read_pipe("git var GIT_EDITOR").strip()
1527 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1529 # If the file was not saved, prompt to see if this patch should
1530 # be skipped. But skip this verification step if configured so.
1531 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1534 # modification time updated means user saved the file
1535 if os.stat(template_file).st_mtime > mtime:
1539 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1545 def get_diff_description(self, editedFiles, filesToAdd):
1547 if os.environ.has_key("P4DIFF"):
1548 del(os.environ["P4DIFF"])
1550 for editedFile in editedFiles:
1551 diff += p4_read_pipe(['diff', '-du',
1552 wildcard_encode(editedFile)])
1556 for newFile in filesToAdd:
1557 newdiff += "==== new file ====\n"
1558 newdiff += "--- /dev/null\n"
1559 newdiff += "+++ %s\n" % newFile
1560 f = open(newFile, "r")
1561 for line in f.readlines():
1562 newdiff += "+" + line
1565 return (diff + newdiff).replace('\r\n', '\n')
1567 def applyCommit(self, id):
1568 """Apply one commit, return True if it succeeded."""
1570 print "Applying", read_pipe(["git", "show", "-s",
1571 "--format=format:%h %s", id])
1573 (p4User, gitEmail) = self.p4UserForCommit(id)
1575 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1577 filesToChangeType = set()
1578 filesToDelete = set()
1580 pureRenameCopy = set()
1581 filesToChangeExecBit = {}
1584 diff = parseDiffTreeEntry(line)
1585 modifier = diff['status']
1589 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1590 filesToChangeExecBit[path] = diff['dst_mode']
1591 editedFiles.add(path)
1592 elif modifier == "A":
1593 filesToAdd.add(path)
1594 filesToChangeExecBit[path] = diff['dst_mode']
1595 if path in filesToDelete:
1596 filesToDelete.remove(path)
1597 elif modifier == "D":
1598 filesToDelete.add(path)
1599 if path in filesToAdd:
1600 filesToAdd.remove(path)
1601 elif modifier == "C":
1602 src, dest = diff['src'], diff['dst']
1603 p4_integrate(src, dest)
1604 pureRenameCopy.add(dest)
1605 if diff['src_sha1'] != diff['dst_sha1']:
1607 pureRenameCopy.discard(dest)
1608 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1610 pureRenameCopy.discard(dest)
1611 filesToChangeExecBit[dest] = diff['dst_mode']
1613 # turn off read-only attribute
1614 os.chmod(dest, stat.S_IWRITE)
1616 editedFiles.add(dest)
1617 elif modifier == "R":
1618 src, dest = diff['src'], diff['dst']
1619 if self.p4HasMoveCommand:
1620 p4_edit(src) # src must be open before move
1621 p4_move(src, dest) # opens for (move/delete, move/add)
1623 p4_integrate(src, dest)
1624 if diff['src_sha1'] != diff['dst_sha1']:
1627 pureRenameCopy.add(dest)
1628 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1629 if not self.p4HasMoveCommand:
1630 p4_edit(dest) # with move: already open, writable
1631 filesToChangeExecBit[dest] = diff['dst_mode']
1632 if not self.p4HasMoveCommand:
1634 os.chmod(dest, stat.S_IWRITE)
1636 filesToDelete.add(src)
1637 editedFiles.add(dest)
1638 elif modifier == "T":
1639 filesToChangeType.add(path)
1641 die("unknown modifier %s for %s" % (modifier, path))
1643 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1644 patchcmd = diffcmd + " | git apply "
1645 tryPatchCmd = patchcmd + "--check -"
1646 applyPatchCmd = patchcmd + "--check --apply -"
1647 patch_succeeded = True
1649 if os.system(tryPatchCmd) != 0:
1650 fixed_rcs_keywords = False
1651 patch_succeeded = False
1652 print "Unfortunately applying the change failed!"
1654 # Patch failed, maybe it's just RCS keyword woes. Look through
1655 # the patch to see if that's possible.
1656 if gitConfigBool("git-p4.attemptRCSCleanup"):
1660 for file in editedFiles | filesToDelete:
1661 # did this file's delta contain RCS keywords?
1662 pattern = p4_keywords_regexp_for_file(file)
1665 # this file is a possibility...look for RCS keywords.
1666 regexp = re.compile(pattern, re.VERBOSE)
1667 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1668 if regexp.search(line):
1670 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1671 kwfiles[file] = pattern
1674 for file in kwfiles:
1676 print "zapping %s with %s" % (line,pattern)
1677 # File is being deleted, so not open in p4. Must
1678 # disable the read-only bit on windows.
1679 if self.isWindows and file not in editedFiles:
1680 os.chmod(file, stat.S_IWRITE)
1681 self.patchRCSKeywords(file, kwfiles[file])
1682 fixed_rcs_keywords = True
1684 if fixed_rcs_keywords:
1685 print "Retrying the patch with RCS keywords cleaned up"
1686 if os.system(tryPatchCmd) == 0:
1687 patch_succeeded = True
1689 if not patch_succeeded:
1690 for f in editedFiles:
1695 # Apply the patch for real, and do add/delete/+x handling.
1697 system(applyPatchCmd)
1699 for f in filesToChangeType:
1700 p4_edit(f, "-t", "auto")
1701 for f in filesToAdd:
1703 for f in filesToDelete:
1707 # Set/clear executable bits
1708 for f in filesToChangeExecBit.keys():
1709 mode = filesToChangeExecBit[f]
1710 setP4ExecBit(f, mode)
1713 # Build p4 change description, starting with the contents
1714 # of the git commit message.
1716 logMessage = extractLogMessageFromGitCommit(id)
1717 logMessage = logMessage.strip()
1718 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1720 template = self.prepareSubmitTemplate()
1721 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1723 if self.preserveUser:
1724 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1726 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1727 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1728 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1729 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1731 separatorLine = "######## everything below this line is just the diff #######\n"
1732 if not self.prepare_p4_only:
1733 submitTemplate += separatorLine
1734 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1736 (handle, fileName) = tempfile.mkstemp()
1737 tmpFile = os.fdopen(handle, "w+b")
1739 submitTemplate = submitTemplate.replace("\n", "\r\n")
1740 tmpFile.write(submitTemplate)
1743 if self.prepare_p4_only:
1745 # Leave the p4 tree prepared, and the submit template around
1746 # and let the user decide what to do next
1749 print "P4 workspace prepared for submission."
1750 print "To submit or revert, go to client workspace"
1751 print " " + self.clientPath
1753 print "To submit, use \"p4 submit\" to write a new description,"
1754 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1755 " \"git p4\"." % fileName
1756 print "You can delete the file \"%s\" when finished." % fileName
1758 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1759 print "To preserve change ownership by user %s, you must\n" \
1760 "do \"p4 change -f <change>\" after submitting and\n" \
1761 "edit the User field."
1763 print "After submitting, renamed files must be re-synced."
1764 print "Invoke \"p4 sync -f\" on each of these files:"
1765 for f in pureRenameCopy:
1769 print "To revert the changes, use \"p4 revert ...\", and delete"
1770 print "the submit template file \"%s\"" % fileName
1772 print "Since the commit adds new files, they must be deleted:"
1773 for f in filesToAdd:
1779 # Let the user edit the change description, then submit it.
1784 if self.edit_template(fileName):
1785 # read the edited message and submit
1786 tmpFile = open(fileName, "rb")
1787 message = tmpFile.read()
1790 message = message.replace("\r\n", "\n")
1791 submitTemplate = message[:message.index(separatorLine)]
1793 p4_write_pipe(['shelve', '-i'], submitTemplate)
1795 p4_write_pipe(['submit', '-i'], submitTemplate)
1796 # The rename/copy happened by applying a patch that created a
1797 # new file. This leaves it writable, which confuses p4.
1798 for f in pureRenameCopy:
1801 if self.preserveUser:
1803 # Get last changelist number. Cannot easily get it from
1804 # the submit command output as the output is
1806 changelist = self.lastP4Changelist()
1807 self.modifyChangelistUser(changelist, p4User)
1813 if not submitted or self.shelve:
1815 print ("Reverting shelved files.")
1817 print ("Submission cancelled, undoing p4 changes.")
1818 for f in editedFiles | filesToDelete:
1820 for f in filesToAdd:
1827 # Export git tags as p4 labels. Create a p4 label and then tag
1829 def exportGitTags(self, gitTags):
1830 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1831 if len(validLabelRegexp) == 0:
1832 validLabelRegexp = defaultLabelRegexp
1833 m = re.compile(validLabelRegexp)
1835 for name in gitTags:
1837 if not m.match(name):
1839 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1842 # Get the p4 commit this corresponds to
1843 logMessage = extractLogMessageFromGitCommit(name)
1844 values = extractSettingsGitLog(logMessage)
1846 if not values.has_key('change'):
1847 # a tag pointing to something not sent to p4; ignore
1849 print "git tag %s does not give a p4 commit" % name
1852 changelist = values['change']
1854 # Get the tag details.
1858 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1861 if re.match(r'tag\s+', l):
1863 elif re.match(r'\s*$', l):
1870 body = ["lightweight tag imported by git p4\n"]
1872 # Create the label - use the same view as the client spec we are using
1873 clientSpec = getClientSpec()
1875 labelTemplate = "Label: %s\n" % name
1876 labelTemplate += "Description:\n"
1878 labelTemplate += "\t" + b + "\n"
1879 labelTemplate += "View:\n"
1880 for depot_side in clientSpec.mappings:
1881 labelTemplate += "\t%s\n" % depot_side
1884 print "Would create p4 label %s for tag" % name
1885 elif self.prepare_p4_only:
1886 print "Not creating p4 label %s for tag due to option" \
1887 " --prepare-p4-only" % name
1889 p4_write_pipe(["label", "-i"], labelTemplate)
1892 p4_system(["tag", "-l", name] +
1893 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1896 print "created p4 label for tag %s" % name
1898 def run(self, args):
1900 self.master = currentGitBranch()
1901 elif len(args) == 1:
1902 self.master = args[0]
1903 if not branchExists(self.master):
1904 die("Branch %s does not exist" % self.master)
1909 allowSubmit = gitConfig("git-p4.allowSubmit")
1910 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1911 die("%s is not in git-p4.allowSubmit" % self.master)
1913 [upstream, settings] = findUpstreamBranchPoint()
1914 self.depotPath = settings['depot-paths'][0]
1915 if len(self.origin) == 0:
1916 self.origin = upstream
1918 if self.preserveUser:
1919 if not self.canChangeChangelists():
1920 die("Cannot preserve user names without p4 super-user or admin permissions")
1922 # if not set from the command line, try the config file
1923 if self.conflict_behavior is None:
1924 val = gitConfig("git-p4.conflict")
1926 if val not in self.conflict_behavior_choices:
1927 die("Invalid value '%s' for config git-p4.conflict" % val)
1930 self.conflict_behavior = val
1933 print "Origin branch is " + self.origin
1935 if len(self.depotPath) == 0:
1936 print "Internal error: cannot locate perforce depot path from existing branches"
1939 self.useClientSpec = False
1940 if gitConfigBool("git-p4.useclientspec"):
1941 self.useClientSpec = True
1942 if self.useClientSpec:
1943 self.clientSpecDirs = getClientSpec()
1945 # Check for the existence of P4 branches
1946 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1948 if self.useClientSpec and not branchesDetected:
1949 # all files are relative to the client spec
1950 self.clientPath = getClientRoot()
1952 self.clientPath = p4Where(self.depotPath)
1954 if self.clientPath == "":
1955 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1957 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1958 self.oldWorkingDirectory = os.getcwd()
1960 # ensure the clientPath exists
1961 new_client_dir = False
1962 if not os.path.exists(self.clientPath):
1963 new_client_dir = True
1964 os.makedirs(self.clientPath)
1966 chdir(self.clientPath, is_client_path=True)
1968 print "Would synchronize p4 checkout in %s" % self.clientPath
1970 print "Synchronizing p4 checkout..."
1972 # old one was destroyed, and maybe nobody told p4
1973 p4_sync("...", "-f")
1980 commitish = self.master
1984 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1985 commits.append(line.strip())
1988 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1989 self.checkAuthorship = False
1991 self.checkAuthorship = True
1993 if self.preserveUser:
1994 self.checkValidP4Users(commits)
1997 # Build up a set of options to be passed to diff when
1998 # submitting each commit to p4.
2000 if self.detectRenames:
2001 # command-line -M arg
2002 self.diffOpts = "-M"
2004 # If not explicitly set check the config variable
2005 detectRenames = gitConfig("git-p4.detectRenames")
2007 if detectRenames.lower() == "false" or detectRenames == "":
2009 elif detectRenames.lower() == "true":
2010 self.diffOpts = "-M"
2012 self.diffOpts = "-M%s" % detectRenames
2014 # no command-line arg for -C or --find-copies-harder, just
2016 detectCopies = gitConfig("git-p4.detectCopies")
2017 if detectCopies.lower() == "false" or detectCopies == "":
2019 elif detectCopies.lower() == "true":
2020 self.diffOpts += " -C"
2022 self.diffOpts += " -C%s" % detectCopies
2024 if gitConfigBool("git-p4.detectCopiesHarder"):
2025 self.diffOpts += " --find-copies-harder"
2028 # Apply the commits, one at a time. On failure, ask if should
2029 # continue to try the rest of the patches, or quit.
2034 last = len(commits) - 1
2035 for i, commit in enumerate(commits):
2037 print " ", read_pipe(["git", "show", "-s",
2038 "--format=format:%h %s", commit])
2041 ok = self.applyCommit(commit)
2043 applied.append(commit)
2045 if self.prepare_p4_only and i < last:
2046 print "Processing only the first commit due to option" \
2047 " --prepare-p4-only"
2052 # prompt for what to do, or use the option/variable
2053 if self.conflict_behavior == "ask":
2054 print "What do you want to do?"
2055 response = raw_input("[s]kip this commit but apply"
2056 " the rest, or [q]uit? ")
2059 elif self.conflict_behavior == "skip":
2061 elif self.conflict_behavior == "quit":
2064 die("Unknown conflict_behavior '%s'" %
2065 self.conflict_behavior)
2067 if response[0] == "s":
2068 print "Skipping this commit, but applying the rest"
2070 if response[0] == "q":
2077 chdir(self.oldWorkingDirectory)
2078 shelved_applied = "shelved" if self.shelve else "applied"
2081 elif self.prepare_p4_only:
2083 elif len(commits) == len(applied):
2084 print ("All commits {0}!".format(shelved_applied))
2088 sync.branch = self.branch
2095 if len(applied) == 0:
2096 print ("No commits {0}.".format(shelved_applied))
2098 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2104 print star, read_pipe(["git", "show", "-s",
2105 "--format=format:%h %s", c])
2106 print "You will have to do 'git p4 sync' and rebase."
2108 if gitConfigBool("git-p4.exportLabels"):
2109 self.exportLabels = True
2111 if self.exportLabels:
2112 p4Labels = getP4Labels(self.depotPath)
2113 gitTags = getGitTags()
2115 missingGitTags = gitTags - p4Labels
2116 self.exportGitTags(missingGitTags)
2118 # exit with error unless everything applied perfectly
2119 if len(commits) != len(applied):
2125 """Represent a p4 view ("p4 help views"), and map files in a
2126 repo according to the view."""
2128 def __init__(self, client_name):
2130 self.client_prefix = "//%s/" % client_name
2131 # cache results of "p4 where" to lookup client file locations
2132 self.client_spec_path_cache = {}
2134 def append(self, view_line):
2135 """Parse a view line, splitting it into depot and client
2136 sides. Append to self.mappings, preserving order. This
2137 is only needed for tag creation."""
2139 # Split the view line into exactly two words. P4 enforces
2140 # structure on these lines that simplifies this quite a bit.
2142 # Either or both words may be double-quoted.
2143 # Single quotes do not matter.
2144 # Double-quote marks cannot occur inside the words.
2145 # A + or - prefix is also inside the quotes.
2146 # There are no quotes unless they contain a space.
2147 # The line is already white-space stripped.
2148 # The two words are separated by a single space.
2150 if view_line[0] == '"':
2151 # First word is double quoted. Find its end.
2152 close_quote_index = view_line.find('"', 1)
2153 if close_quote_index <= 0:
2154 die("No first-word closing quote found: %s" % view_line)
2155 depot_side = view_line[1:close_quote_index]
2156 # skip closing quote and space
2157 rhs_index = close_quote_index + 1 + 1
2159 space_index = view_line.find(" ")
2160 if space_index <= 0:
2161 die("No word-splitting space found: %s" % view_line)
2162 depot_side = view_line[0:space_index]
2163 rhs_index = space_index + 1
2165 # prefix + means overlay on previous mapping
2166 if depot_side.startswith("+"):
2167 depot_side = depot_side[1:]
2169 # prefix - means exclude this path, leave out of mappings
2171 if depot_side.startswith("-"):
2173 depot_side = depot_side[1:]
2176 self.mappings.append(depot_side)
2178 def convert_client_path(self, clientFile):
2179 # chop off //client/ part to make it relative
2180 if not clientFile.startswith(self.client_prefix):
2181 die("No prefix '%s' on clientFile '%s'" %
2182 (self.client_prefix, clientFile))
2183 return clientFile[len(self.client_prefix):]
2185 def update_client_spec_path_cache(self, files):
2186 """ Caching file paths by "p4 where" batch query """
2188 # List depot file paths exclude that already cached
2189 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2191 if len(fileArgs) == 0:
2192 return # All files in cache
2194 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2195 for res in where_result:
2196 if "code" in res and res["code"] == "error":
2197 # assume error is "... file(s) not in client view"
2199 if "clientFile" not in res:
2200 die("No clientFile in 'p4 where' output")
2202 # it will list all of them, but only one not unmap-ped
2204 if gitConfigBool("core.ignorecase"):
2205 res['depotFile'] = res['depotFile'].lower()
2206 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2208 # not found files or unmap files set to ""
2209 for depotFile in fileArgs:
2210 if gitConfigBool("core.ignorecase"):
2211 depotFile = depotFile.lower()
2212 if depotFile not in self.client_spec_path_cache:
2213 self.client_spec_path_cache[depotFile] = ""
2215 def map_in_client(self, depot_path):
2216 """Return the relative location in the client where this
2217 depot file should live. Returns "" if the file should
2218 not be mapped in the client."""
2220 if gitConfigBool("core.ignorecase"):
2221 depot_path = depot_path.lower()
2223 if depot_path in self.client_spec_path_cache:
2224 return self.client_spec_path_cache[depot_path]
2226 die( "Error: %s is not found in client spec path" % depot_path )
2229 class P4Sync(Command, P4UserMap):
2230 delete_actions = ( "delete", "move/delete", "purge" )
2233 Command.__init__(self)
2234 P4UserMap.__init__(self)
2236 optparse.make_option("--branch", dest="branch"),
2237 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2238 optparse.make_option("--changesfile", dest="changesFile"),
2239 optparse.make_option("--silent", dest="silent", action="store_true"),
2240 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2241 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2242 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2243 help="Import into refs/heads/ , not refs/remotes"),
2244 optparse.make_option("--max-changes", dest="maxChanges",
2245 help="Maximum number of changes to import"),
2246 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2247 help="Internal block size to use when iteratively calling p4 changes"),
2248 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2249 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2250 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2251 help="Only sync files that are included in the Perforce Client Spec"),
2252 optparse.make_option("-/", dest="cloneExclude",
2253 action="append", type="string",
2254 help="exclude depot path"),
2256 self.description = """Imports from Perforce into a git repository.\n
2258 //depot/my/project/ -- to import the current head
2259 //depot/my/project/@all -- to import everything
2260 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2262 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2264 self.usage += " //depot/path[@revRange]"
2266 self.createdBranches = set()
2267 self.committedChanges = set()
2269 self.detectBranches = False
2270 self.detectLabels = False
2271 self.importLabels = False
2272 self.changesFile = ""
2273 self.syncWithOrigin = True
2274 self.importIntoRemotes = True
2275 self.maxChanges = ""
2276 self.changes_block_size = None
2277 self.keepRepoPath = False
2278 self.depotPaths = None
2279 self.p4BranchesInGit = []
2280 self.cloneExclude = []
2281 self.useClientSpec = False
2282 self.useClientSpec_from_options = False
2283 self.clientSpecDirs = None
2284 self.tempBranches = []
2285 self.tempBranchLocation = "refs/git-p4-tmp"
2286 self.largeFileSystem = None
2288 if gitConfig('git-p4.largeFileSystem'):
2289 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2290 self.largeFileSystem = largeFileSystemConstructor(
2291 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2294 if gitConfig("git-p4.syncFromOrigin") == "false":
2295 self.syncWithOrigin = False
2297 # This is required for the "append" cloneExclude action
2298 def ensure_value(self, attr, value):
2299 if not hasattr(self, attr) or getattr(self, attr) is None:
2300 setattr(self, attr, value)
2301 return getattr(self, attr)
2303 # Force a checkpoint in fast-import and wait for it to finish
2304 def checkpoint(self):
2305 self.gitStream.write("checkpoint\n\n")
2306 self.gitStream.write("progress checkpoint\n\n")
2307 out = self.gitOutput.readline()
2309 print "checkpoint finished: " + out
2311 def extractFilesFromCommit(self, commit):
2312 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2313 for path in self.cloneExclude]
2316 while commit.has_key("depotFile%s" % fnum):
2317 path = commit["depotFile%s" % fnum]
2319 if [p for p in self.cloneExclude
2320 if p4PathStartsWith(path, p)]:
2323 found = [p for p in self.depotPaths
2324 if p4PathStartsWith(path, p)]
2331 file["rev"] = commit["rev%s" % fnum]
2332 file["action"] = commit["action%s" % fnum]
2333 file["type"] = commit["type%s" % fnum]
2338 def extractJobsFromCommit(self, commit):
2341 while commit.has_key("job%s" % jnum):
2342 job = commit["job%s" % jnum]
2347 def stripRepoPath(self, path, prefixes):
2348 """When streaming files, this is called to map a p4 depot path
2349 to where it should go in git. The prefixes are either
2350 self.depotPaths, or self.branchPrefixes in the case of
2351 branch detection."""
2353 if self.useClientSpec:
2354 # branch detection moves files up a level (the branch name)
2355 # from what client spec interpretation gives
2356 path = self.clientSpecDirs.map_in_client(path)
2357 if self.detectBranches:
2358 for b in self.knownBranches:
2359 if path.startswith(b + "/"):
2360 path = path[len(b)+1:]
2362 elif self.keepRepoPath:
2363 # Preserve everything in relative path name except leading
2364 # //depot/; just look at first prefix as they all should
2365 # be in the same depot.
2366 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2367 if p4PathStartsWith(path, depot):
2368 path = path[len(depot):]
2372 if p4PathStartsWith(path, p):
2373 path = path[len(p):]
2376 path = wildcard_decode(path)
2379 def splitFilesIntoBranches(self, commit):
2380 """Look at each depotFile in the commit to figure out to what
2381 branch it belongs."""
2383 if self.clientSpecDirs:
2384 files = self.extractFilesFromCommit(commit)
2385 self.clientSpecDirs.update_client_spec_path_cache(files)
2389 while commit.has_key("depotFile%s" % fnum):
2390 path = commit["depotFile%s" % fnum]
2391 found = [p for p in self.depotPaths
2392 if p4PathStartsWith(path, p)]
2399 file["rev"] = commit["rev%s" % fnum]
2400 file["action"] = commit["action%s" % fnum]
2401 file["type"] = commit["type%s" % fnum]
2404 # start with the full relative path where this file would
2406 if self.useClientSpec:
2407 relPath = self.clientSpecDirs.map_in_client(path)
2409 relPath = self.stripRepoPath(path, self.depotPaths)
2411 for branch in self.knownBranches.keys():
2412 # add a trailing slash so that a commit into qt/4.2foo
2413 # doesn't end up in qt/4.2, e.g.
2414 if relPath.startswith(branch + "/"):
2415 if branch not in branches:
2416 branches[branch] = []
2417 branches[branch].append(file)
2422 def writeToGitStream(self, gitMode, relPath, contents):
2423 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2424 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2426 self.gitStream.write(d)
2427 self.gitStream.write('\n')
2429 # output one file from the P4 stream
2430 # - helper for streamP4Files
2432 def streamOneP4File(self, file, contents):
2433 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2435 size = int(self.stream_file['fileSize'])
2436 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2439 (type_base, type_mods) = split_p4_type(file["type"])
2442 if "x" in type_mods:
2444 if type_base == "symlink":
2446 # p4 print on a symlink sometimes contains "target\n";
2447 # if it does, remove the newline
2448 data = ''.join(contents)
2450 # Some version of p4 allowed creating a symlink that pointed
2451 # to nothing. This causes p4 errors when checking out such
2452 # a change, and errors here too. Work around it by ignoring
2453 # the bad symlink; hopefully a future change fixes it.
2454 print "\nIgnoring empty symlink in %s" % file['depotFile']
2456 elif data[-1] == '\n':
2457 contents = [data[:-1]]
2461 if type_base == "utf16":
2462 # p4 delivers different text in the python output to -G
2463 # than it does when using "print -o", or normal p4 client
2464 # operations. utf16 is converted to ascii or utf8, perhaps.
2465 # But ascii text saved as -t utf16 is completely mangled.
2466 # Invoke print -o to get the real contents.
2468 # On windows, the newlines will always be mangled by print, so put
2469 # them back too. This is not needed to the cygwin windows version,
2470 # just the native "NT" type.
2473 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2474 except Exception as e:
2475 if 'Translation of file content failed' in str(e):
2476 type_base = 'binary'
2480 if p4_version_string().find('/NT') >= 0:
2481 text = text.replace('\r\n', '\n')
2484 if type_base == "apple":
2485 # Apple filetype files will be streamed as a concatenation of
2486 # its appledouble header and the contents. This is useless
2487 # on both macs and non-macs. If using "print -q -o xx", it
2488 # will create "xx" with the data, and "%xx" with the header.
2489 # This is also not very useful.
2491 # Ideally, someday, this script can learn how to generate
2492 # appledouble files directly and import those to git, but
2493 # non-mac machines can never find a use for apple filetype.
2494 print "\nIgnoring apple filetype file %s" % file['depotFile']
2497 # Note that we do not try to de-mangle keywords on utf16 files,
2498 # even though in theory somebody may want that.
2499 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2501 regexp = re.compile(pattern, re.VERBOSE)
2502 text = ''.join(contents)
2503 text = regexp.sub(r'$\1$', text)
2507 relPath.decode('ascii')
2510 if gitConfig('git-p4.pathEncoding'):
2511 encoding = gitConfig('git-p4.pathEncoding')
2512 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2514 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2516 if self.largeFileSystem:
2517 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2519 self.writeToGitStream(git_mode, relPath, contents)
2521 def streamOneP4Deletion(self, file):
2522 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2524 sys.stdout.write("delete %s\n" % relPath)
2526 self.gitStream.write("D %s\n" % relPath)
2528 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2529 self.largeFileSystem.removeLargeFile(relPath)
2531 # handle another chunk of streaming data
2532 def streamP4FilesCb(self, marshalled):
2534 # catch p4 errors and complain
2536 if "code" in marshalled:
2537 if marshalled["code"] == "error":
2538 if "data" in marshalled:
2539 err = marshalled["data"].rstrip()
2541 if not err and 'fileSize' in self.stream_file:
2542 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2543 if required_bytes > 0:
2544 err = 'Not enough space left on %s! Free at least %i MB.' % (
2545 os.getcwd(), required_bytes/1024/1024
2550 if self.stream_have_file_info:
2551 if "depotFile" in self.stream_file:
2552 f = self.stream_file["depotFile"]
2553 # force a failure in fast-import, else an empty
2554 # commit will be made
2555 self.gitStream.write("\n")
2556 self.gitStream.write("die-now\n")
2557 self.gitStream.close()
2558 # ignore errors, but make sure it exits first
2559 self.importProcess.wait()
2561 die("Error from p4 print for %s: %s" % (f, err))
2563 die("Error from p4 print: %s" % err)
2565 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2566 # start of a new file - output the old one first
2567 self.streamOneP4File(self.stream_file, self.stream_contents)
2568 self.stream_file = {}
2569 self.stream_contents = []
2570 self.stream_have_file_info = False
2572 # pick up the new file information... for the
2573 # 'data' field we need to append to our array
2574 for k in marshalled.keys():
2576 if 'streamContentSize' not in self.stream_file:
2577 self.stream_file['streamContentSize'] = 0
2578 self.stream_file['streamContentSize'] += len(marshalled['data'])
2579 self.stream_contents.append(marshalled['data'])
2581 self.stream_file[k] = marshalled[k]
2584 'streamContentSize' in self.stream_file and
2585 'fileSize' in self.stream_file and
2586 'depotFile' in self.stream_file):
2587 size = int(self.stream_file["fileSize"])
2589 progress = 100*self.stream_file['streamContentSize']/size
2590 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2593 self.stream_have_file_info = True
2595 # Stream directly from "p4 files" into "git fast-import"
2596 def streamP4Files(self, files):
2602 filesForCommit.append(f)
2603 if f['action'] in self.delete_actions:
2604 filesToDelete.append(f)
2606 filesToRead.append(f)
2609 for f in filesToDelete:
2610 self.streamOneP4Deletion(f)
2612 if len(filesToRead) > 0:
2613 self.stream_file = {}
2614 self.stream_contents = []
2615 self.stream_have_file_info = False
2617 # curry self argument
2618 def streamP4FilesCbSelf(entry):
2619 self.streamP4FilesCb(entry)
2621 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2623 p4CmdList(["-x", "-", "print"],
2625 cb=streamP4FilesCbSelf)
2628 if self.stream_file.has_key('depotFile'):
2629 self.streamOneP4File(self.stream_file, self.stream_contents)
2631 def make_email(self, userid):
2632 if userid in self.users:
2633 return self.users[userid]
2635 return "%s <a@b>" % userid
2637 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2638 """ Stream a p4 tag.
2639 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2643 print "writing tag %s for commit %s" % (labelName, commit)
2644 gitStream.write("tag %s\n" % labelName)
2645 gitStream.write("from %s\n" % commit)
2647 if labelDetails.has_key('Owner'):
2648 owner = labelDetails["Owner"]
2652 # Try to use the owner of the p4 label, or failing that,
2653 # the current p4 user id.
2655 email = self.make_email(owner)
2657 email = self.make_email(self.p4UserId())
2658 tagger = "%s %s %s" % (email, epoch, self.tz)
2660 gitStream.write("tagger %s\n" % tagger)
2662 print "labelDetails=",labelDetails
2663 if labelDetails.has_key('Description'):
2664 description = labelDetails['Description']
2666 description = 'Label from git p4'
2668 gitStream.write("data %d\n" % len(description))
2669 gitStream.write(description)
2670 gitStream.write("\n")
2672 def inClientSpec(self, path):
2673 if not self.clientSpecDirs:
2675 inClientSpec = self.clientSpecDirs.map_in_client(path)
2676 if not inClientSpec and self.verbose:
2677 print('Ignoring file outside of client spec: {0}'.format(path))
2680 def hasBranchPrefix(self, path):
2681 if not self.branchPrefixes:
2683 hasPrefix = [p for p in self.branchPrefixes
2684 if p4PathStartsWith(path, p)]
2685 if not hasPrefix and self.verbose:
2686 print('Ignoring file outside of prefix: {0}'.format(path))
2689 def commit(self, details, files, branch, parent = ""):
2690 epoch = details["time"]
2691 author = details["user"]
2692 jobs = self.extractJobsFromCommit(details)
2695 print('commit into {0}'.format(branch))
2697 if self.clientSpecDirs:
2698 self.clientSpecDirs.update_client_spec_path_cache(files)
2700 files = [f for f in files
2701 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2703 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2704 print('Ignoring revision {0} as it would produce an empty commit.'
2705 .format(details['change']))
2708 self.gitStream.write("commit %s\n" % branch)
2709 self.gitStream.write("mark :%s\n" % details["change"])
2710 self.committedChanges.add(int(details["change"]))
2712 if author not in self.users:
2713 self.getUserMapFromPerforceServer()
2714 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2716 self.gitStream.write("committer %s\n" % committer)
2718 self.gitStream.write("data <<EOT\n")
2719 self.gitStream.write(details["desc"])
2721 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2722 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2723 (','.join(self.branchPrefixes), details["change"]))
2724 if len(details['options']) > 0:
2725 self.gitStream.write(": options = %s" % details['options'])
2726 self.gitStream.write("]\nEOT\n\n")
2730 print "parent %s" % parent
2731 self.gitStream.write("from %s\n" % parent)
2733 self.streamP4Files(files)
2734 self.gitStream.write("\n")
2736 change = int(details["change"])
2738 if self.labels.has_key(change):
2739 label = self.labels[change]
2740 labelDetails = label[0]
2741 labelRevisions = label[1]
2743 print "Change %s is labelled %s" % (change, labelDetails)
2745 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2746 for p in self.branchPrefixes])
2748 if len(files) == len(labelRevisions):
2752 if info["action"] in self.delete_actions:
2754 cleanedFiles[info["depotFile"]] = info["rev"]
2756 if cleanedFiles == labelRevisions:
2757 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2761 print ("Tag %s does not match with change %s: files do not match."
2762 % (labelDetails["label"], change))
2766 print ("Tag %s does not match with change %s: file count is different."
2767 % (labelDetails["label"], change))
2769 # Build a dictionary of changelists and labels, for "detect-labels" option.
2770 def getLabels(self):
2773 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2774 if len(l) > 0 and not self.silent:
2775 print "Finding files belonging to labels in %s" % `self.depotPaths`
2778 label = output["label"]
2782 print "Querying files for label %s" % label
2783 for file in p4CmdList(["files"] +
2784 ["%s...@%s" % (p, label)
2785 for p in self.depotPaths]):
2786 revisions[file["depotFile"]] = file["rev"]
2787 change = int(file["change"])
2788 if change > newestChange:
2789 newestChange = change
2791 self.labels[newestChange] = [output, revisions]
2794 print "Label changes: %s" % self.labels.keys()
2796 # Import p4 labels as git tags. A direct mapping does not
2797 # exist, so assume that if all the files are at the same revision
2798 # then we can use that, or it's something more complicated we should
2800 def importP4Labels(self, stream, p4Labels):
2802 print "import p4 labels: " + ' '.join(p4Labels)
2804 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2805 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2806 if len(validLabelRegexp) == 0:
2807 validLabelRegexp = defaultLabelRegexp
2808 m = re.compile(validLabelRegexp)
2810 for name in p4Labels:
2813 if not m.match(name):
2815 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2818 if name in ignoredP4Labels:
2821 labelDetails = p4CmdList(['label', "-o", name])[0]
2823 # get the most recent changelist for each file in this label
2824 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2825 for p in self.depotPaths])
2827 if change.has_key('change'):
2828 # find the corresponding git commit; take the oldest commit
2829 changelist = int(change['change'])
2830 if changelist in self.committedChanges:
2831 gitCommit = ":%d" % changelist # use a fast-import mark
2834 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2835 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2836 if len(gitCommit) == 0:
2837 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2840 gitCommit = gitCommit.strip()
2843 # Convert from p4 time format
2845 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2847 print "Could not convert label time %s" % labelDetails['Update']
2850 when = int(time.mktime(tmwhen))
2851 self.streamTag(stream, name, labelDetails, gitCommit, when)
2853 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2856 print "Label %s has no changelists - possibly deleted?" % name
2859 # We can't import this label; don't try again as it will get very
2860 # expensive repeatedly fetching all the files for labels that will
2861 # never be imported. If the label is moved in the future, the
2862 # ignore will need to be removed manually.
2863 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2865 def guessProjectName(self):
2866 for p in self.depotPaths:
2869 p = p[p.strip().rfind("/") + 1:]
2870 if not p.endswith("/"):
2874 def getBranchMapping(self):
2875 lostAndFoundBranches = set()
2877 user = gitConfig("git-p4.branchUser")
2879 command = "branches -u %s" % user
2881 command = "branches"
2883 for info in p4CmdList(command):
2884 details = p4Cmd(["branch", "-o", info["branch"]])
2886 while details.has_key("View%s" % viewIdx):
2887 paths = details["View%s" % viewIdx].split(" ")
2888 viewIdx = viewIdx + 1
2889 # require standard //depot/foo/... //depot/bar/... mapping
2890 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2893 destination = paths[1]
2895 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2896 source = source[len(self.depotPaths[0]):-4]
2897 destination = destination[len(self.depotPaths[0]):-4]
2899 if destination in self.knownBranches:
2901 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2902 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2905 self.knownBranches[destination] = source
2907 lostAndFoundBranches.discard(destination)
2909 if source not in self.knownBranches:
2910 lostAndFoundBranches.add(source)
2912 # Perforce does not strictly require branches to be defined, so we also
2913 # check git config for a branch list.
2915 # Example of branch definition in git config file:
2917 # branchList=main:branchA
2918 # branchList=main:branchB
2919 # branchList=branchA:branchC
2920 configBranches = gitConfigList("git-p4.branchList")
2921 for branch in configBranches:
2923 (source, destination) = branch.split(":")
2924 self.knownBranches[destination] = source
2926 lostAndFoundBranches.discard(destination)
2928 if source not in self.knownBranches:
2929 lostAndFoundBranches.add(source)
2932 for branch in lostAndFoundBranches:
2933 self.knownBranches[branch] = branch
2935 def getBranchMappingFromGitBranches(self):
2936 branches = p4BranchesInGit(self.importIntoRemotes)
2937 for branch in branches.keys():
2938 if branch == "master":
2941 branch = branch[len(self.projectName):]
2942 self.knownBranches[branch] = branch
2944 def updateOptionDict(self, d):
2946 if self.keepRepoPath:
2947 option_keys['keepRepoPath'] = 1
2949 d["options"] = ' '.join(sorted(option_keys.keys()))
2951 def readOptions(self, d):
2952 self.keepRepoPath = (d.has_key('options')
2953 and ('keepRepoPath' in d['options']))
2955 def gitRefForBranch(self, branch):
2956 if branch == "main":
2957 return self.refPrefix + "master"
2959 if len(branch) <= 0:
2962 return self.refPrefix + self.projectName + branch
2964 def gitCommitByP4Change(self, ref, change):
2966 print "looking in ref " + ref + " for change %s using bisect..." % change
2969 latestCommit = parseRevision(ref)
2973 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2974 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2979 log = extractLogMessageFromGitCommit(next)
2980 settings = extractSettingsGitLog(log)
2981 currentChange = int(settings['change'])
2983 print "current change %s" % currentChange
2985 if currentChange == change:
2987 print "found %s" % next
2990 if currentChange < change:
2991 earliestCommit = "^%s" % next
2993 latestCommit = "%s" % next
2997 def importNewBranch(self, branch, maxChange):
2998 # make fast-import flush all changes to disk and update the refs using the checkpoint
2999 # command so that we can try to find the branch parent in the git history
3000 self.gitStream.write("checkpoint\n\n");
3001 self.gitStream.flush();
3002 branchPrefix = self.depotPaths[0] + branch + "/"
3003 range = "@1,%s" % maxChange
3004 #print "prefix" + branchPrefix
3005 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3006 if len(changes) <= 0:
3008 firstChange = changes[0]
3009 #print "first change in branch: %s" % firstChange
3010 sourceBranch = self.knownBranches[branch]
3011 sourceDepotPath = self.depotPaths[0] + sourceBranch
3012 sourceRef = self.gitRefForBranch(sourceBranch)
3013 #print "source " + sourceBranch
3015 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3016 #print "branch parent: %s" % branchParentChange
3017 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3018 if len(gitParent) > 0:
3019 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3020 #print "parent git commit: %s" % gitParent
3022 self.importChanges(changes)
3025 def searchParent(self, parent, branch, target):
3027 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3028 "--no-merges", parent]):
3030 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3033 print "Found parent of %s in commit %s" % (branch, blob)
3040 def importChanges(self, changes):
3042 for change in changes:
3043 description = p4_describe(change)
3044 self.updateOptionDict(description)
3047 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3052 if self.detectBranches:
3053 branches = self.splitFilesIntoBranches(description)
3054 for branch in branches.keys():
3056 branchPrefix = self.depotPaths[0] + branch + "/"
3057 self.branchPrefixes = [ branchPrefix ]
3061 filesForCommit = branches[branch]
3064 print "branch is %s" % branch
3066 self.updatedBranches.add(branch)
3068 if branch not in self.createdBranches:
3069 self.createdBranches.add(branch)
3070 parent = self.knownBranches[branch]
3071 if parent == branch:
3074 fullBranch = self.projectName + branch
3075 if fullBranch not in self.p4BranchesInGit:
3077 print("\n Importing new branch %s" % fullBranch);
3078 if self.importNewBranch(branch, change - 1):
3080 self.p4BranchesInGit.append(fullBranch)
3082 print("\n Resuming with change %s" % change);
3085 print "parent determined through known branches: %s" % parent
3087 branch = self.gitRefForBranch(branch)
3088 parent = self.gitRefForBranch(parent)
3091 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3093 if len(parent) == 0 and branch in self.initialParents:
3094 parent = self.initialParents[branch]
3095 del self.initialParents[branch]
3099 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3101 print "Creating temporary branch: " + tempBranch
3102 self.commit(description, filesForCommit, tempBranch)
3103 self.tempBranches.append(tempBranch)
3105 blob = self.searchParent(parent, branch, tempBranch)
3107 self.commit(description, filesForCommit, branch, blob)
3110 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3111 self.commit(description, filesForCommit, branch, parent)
3113 files = self.extractFilesFromCommit(description)
3114 self.commit(description, files, self.branch,
3116 # only needed once, to connect to the previous commit
3117 self.initialParent = ""
3119 print self.gitError.read()
3122 def importHeadRevision(self, revision):
3123 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3126 details["user"] = "git perforce import user"
3127 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3128 % (' '.join(self.depotPaths), revision))
3129 details["change"] = revision
3133 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3135 for info in p4CmdList(["files"] + fileArgs):
3137 if 'code' in info and info['code'] == 'error':
3138 sys.stderr.write("p4 returned an error: %s\n"
3140 if info['data'].find("must refer to client") >= 0:
3141 sys.stderr.write("This particular p4 error is misleading.\n")
3142 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3143 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3145 if 'p4ExitCode' in info:
3146 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3150 change = int(info["change"])
3151 if change > newestRevision:
3152 newestRevision = change
3154 if info["action"] in self.delete_actions:
3155 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3156 #fileCnt = fileCnt + 1
3159 for prop in ["depotFile", "rev", "action", "type" ]:
3160 details["%s%s" % (prop, fileCnt)] = info[prop]
3162 fileCnt = fileCnt + 1
3164 details["change"] = newestRevision
3166 # Use time from top-most change so that all git p4 clones of
3167 # the same p4 repo have the same commit SHA1s.
3168 res = p4_describe(newestRevision)
3169 details["time"] = res["time"]
3171 self.updateOptionDict(details)
3173 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3175 print "IO error with git fast-import. Is your git version recent enough?"
3176 print self.gitError.read()
3179 def run(self, args):
3180 self.depotPaths = []
3181 self.changeRange = ""
3182 self.previousDepotPaths = []
3183 self.hasOrigin = False
3185 # map from branch depot path to parent branch
3186 self.knownBranches = {}
3187 self.initialParents = {}
3189 if self.importIntoRemotes:
3190 self.refPrefix = "refs/remotes/p4/"
3192 self.refPrefix = "refs/heads/p4/"
3194 if self.syncWithOrigin:
3195 self.hasOrigin = originP4BranchesExist()
3198 print 'Syncing with origin first, using "git fetch origin"'
3199 system("git fetch origin")
3201 branch_arg_given = bool(self.branch)
3202 if len(self.branch) == 0:
3203 self.branch = self.refPrefix + "master"
3204 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3205 system("git update-ref %s refs/heads/p4" % self.branch)
3206 system("git branch -D p4")
3208 # accept either the command-line option, or the configuration variable
3209 if self.useClientSpec:
3210 # will use this after clone to set the variable
3211 self.useClientSpec_from_options = True
3213 if gitConfigBool("git-p4.useclientspec"):
3214 self.useClientSpec = True
3215 if self.useClientSpec:
3216 self.clientSpecDirs = getClientSpec()
3218 # TODO: should always look at previous commits,
3219 # merge with previous imports, if possible.
3222 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3224 # branches holds mapping from branch name to sha1
3225 branches = p4BranchesInGit(self.importIntoRemotes)
3227 # restrict to just this one, disabling detect-branches
3228 if branch_arg_given:
3229 short = self.branch.split("/")[-1]
3230 if short in branches:
3231 self.p4BranchesInGit = [ short ]
3233 self.p4BranchesInGit = branches.keys()
3235 if len(self.p4BranchesInGit) > 1:
3237 print "Importing from/into multiple branches"
3238 self.detectBranches = True
3239 for branch in branches.keys():
3240 self.initialParents[self.refPrefix + branch] = \
3244 print "branches: %s" % self.p4BranchesInGit
3247 for branch in self.p4BranchesInGit:
3248 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3250 settings = extractSettingsGitLog(logMsg)
3252 self.readOptions(settings)
3253 if (settings.has_key('depot-paths')
3254 and settings.has_key ('change')):
3255 change = int(settings['change']) + 1
3256 p4Change = max(p4Change, change)
3258 depotPaths = sorted(settings['depot-paths'])
3259 if self.previousDepotPaths == []:
3260 self.previousDepotPaths = depotPaths
3263 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3264 prev_list = prev.split("/")
3265 cur_list = cur.split("/")
3266 for i in range(0, min(len(cur_list), len(prev_list))):
3267 if cur_list[i] <> prev_list[i]:
3271 paths.append ("/".join(cur_list[:i + 1]))
3273 self.previousDepotPaths = paths
3276 self.depotPaths = sorted(self.previousDepotPaths)
3277 self.changeRange = "@%s,#head" % p4Change
3278 if not self.silent and not self.detectBranches:
3279 print "Performing incremental import into %s git branch" % self.branch
3281 # accept multiple ref name abbreviations:
3282 # refs/foo/bar/branch -> use it exactly
3283 # p4/branch -> prepend refs/remotes/ or refs/heads/
3284 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3285 if not self.branch.startswith("refs/"):
3286 if self.importIntoRemotes:
3287 prepend = "refs/remotes/"
3289 prepend = "refs/heads/"
3290 if not self.branch.startswith("p4/"):
3292 self.branch = prepend + self.branch
3294 if len(args) == 0 and self.depotPaths:
3296 print "Depot paths: %s" % ' '.join(self.depotPaths)
3298 if self.depotPaths and self.depotPaths != args:
3299 print ("previous import used depot path %s and now %s was specified. "
3300 "This doesn't work!" % (' '.join (self.depotPaths),
3304 self.depotPaths = sorted(args)
3309 # Make sure no revision specifiers are used when --changesfile
3311 bad_changesfile = False
3312 if len(self.changesFile) > 0:
3313 for p in self.depotPaths:
3314 if p.find("@") >= 0 or p.find("#") >= 0:
3315 bad_changesfile = True
3318 die("Option --changesfile is incompatible with revision specifiers")
3321 for p in self.depotPaths:
3322 if p.find("@") != -1:
3323 atIdx = p.index("@")
3324 self.changeRange = p[atIdx:]
3325 if self.changeRange == "@all":
3326 self.changeRange = ""
3327 elif ',' not in self.changeRange:
3328 revision = self.changeRange
3329 self.changeRange = ""
3331 elif p.find("#") != -1:
3332 hashIdx = p.index("#")
3333 revision = p[hashIdx:]
3335 elif self.previousDepotPaths == []:
3336 # pay attention to changesfile, if given, else import
3337 # the entire p4 tree at the head revision
3338 if len(self.changesFile) == 0:
3341 p = re.sub ("\.\.\.$", "", p)
3342 if not p.endswith("/"):
3347 self.depotPaths = newPaths
3349 # --detect-branches may change this for each branch
3350 self.branchPrefixes = self.depotPaths
3352 self.loadUserMapFromCache()
3354 if self.detectLabels:
3357 if self.detectBranches:
3358 ## FIXME - what's a P4 projectName ?
3359 self.projectName = self.guessProjectName()
3362 self.getBranchMappingFromGitBranches()
3364 self.getBranchMapping()
3366 print "p4-git branches: %s" % self.p4BranchesInGit
3367 print "initial parents: %s" % self.initialParents
3368 for b in self.p4BranchesInGit:
3372 b = b[len(self.projectName):]
3373 self.createdBranches.add(b)
3375 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3377 self.importProcess = subprocess.Popen(["git", "fast-import"],
3378 stdin=subprocess.PIPE,
3379 stdout=subprocess.PIPE,
3380 stderr=subprocess.PIPE);
3381 self.gitOutput = self.importProcess.stdout
3382 self.gitStream = self.importProcess.stdin
3383 self.gitError = self.importProcess.stderr
3386 self.importHeadRevision(revision)
3390 if len(self.changesFile) > 0:
3391 output = open(self.changesFile).readlines()
3394 changeSet.add(int(line))
3396 for change in changeSet:
3397 changes.append(change)
3401 # catch "git p4 sync" with no new branches, in a repo that
3402 # does not have any existing p4 branches
3404 if not self.p4BranchesInGit:
3405 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3407 # The default branch is master, unless --branch is used to
3408 # specify something else. Make sure it exists, or complain
3409 # nicely about how to use --branch.
3410 if not self.detectBranches:
3411 if not branch_exists(self.branch):
3412 if branch_arg_given:
3413 die("Error: branch %s does not exist." % self.branch)
3415 die("Error: no branch %s; perhaps specify one with --branch." %
3419 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3421 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3423 if len(self.maxChanges) > 0:
3424 changes = changes[:min(int(self.maxChanges), len(changes))]
3426 if len(changes) == 0:
3428 print "No changes to import!"
3430 if not self.silent and not self.detectBranches:
3431 print "Import destination: %s" % self.branch
3433 self.updatedBranches = set()
3435 if not self.detectBranches:
3437 # start a new branch
3438 self.initialParent = ""
3440 # build on a previous revision
3441 self.initialParent = parseRevision(self.branch)
3443 self.importChanges(changes)
3447 if len(self.updatedBranches) > 0:
3448 sys.stdout.write("Updated branches: ")
3449 for b in self.updatedBranches:
3450 sys.stdout.write("%s " % b)
3451 sys.stdout.write("\n")
3453 if gitConfigBool("git-p4.importLabels"):
3454 self.importLabels = True
3456 if self.importLabels:
3457 p4Labels = getP4Labels(self.depotPaths)
3458 gitTags = getGitTags()
3460 missingP4Labels = p4Labels - gitTags
3461 self.importP4Labels(self.gitStream, missingP4Labels)
3463 self.gitStream.close()
3464 if self.importProcess.wait() != 0:
3465 die("fast-import failed: %s" % self.gitError.read())
3466 self.gitOutput.close()
3467 self.gitError.close()
3469 # Cleanup temporary branches created during import
3470 if self.tempBranches != []:
3471 for branch in self.tempBranches:
3472 read_pipe("git update-ref -d %s" % branch)
3473 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3475 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3476 # a convenient shortcut refname "p4".
3477 if self.importIntoRemotes:
3478 head_ref = self.refPrefix + "HEAD"
3479 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3480 system(["git", "symbolic-ref", head_ref, self.branch])
3484 class P4Rebase(Command):
3486 Command.__init__(self)
3488 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3490 self.importLabels = False
3491 self.description = ("Fetches the latest revision from perforce and "
3492 + "rebases the current work (branch) against it")
3494 def run(self, args):
3496 sync.importLabels = self.importLabels
3499 return self.rebase()
3502 if os.system("git update-index --refresh") != 0:
3503 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3504 if len(read_pipe("git diff-index HEAD --")) > 0:
3505 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3507 [upstream, settings] = findUpstreamBranchPoint()
3508 if len(upstream) == 0:
3509 die("Cannot find upstream branchpoint for rebase")
3511 # the branchpoint may be p4/foo~3, so strip off the parent
3512 upstream = re.sub("~[0-9]+$", "", upstream)
3514 print "Rebasing the current branch onto %s" % upstream
3515 oldHead = read_pipe("git rev-parse HEAD").strip()
3516 system("git rebase %s" % upstream)
3517 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3520 class P4Clone(P4Sync):
3522 P4Sync.__init__(self)
3523 self.description = "Creates a new git repository and imports from Perforce into it"
3524 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3526 optparse.make_option("--destination", dest="cloneDestination",
3527 action='store', default=None,
3528 help="where to leave result of the clone"),
3529 optparse.make_option("--bare", dest="cloneBare",
3530 action="store_true", default=False),
3532 self.cloneDestination = None
3533 self.needsGit = False
3534 self.cloneBare = False
3536 def defaultDestination(self, args):
3537 ## TODO: use common prefix of args?
3539 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3540 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3541 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3542 depotDir = re.sub(r"/$", "", depotDir)
3543 return os.path.split(depotDir)[1]
3545 def run(self, args):
3549 if self.keepRepoPath and not self.cloneDestination:
3550 sys.stderr.write("Must specify destination for --keep-path\n")
3555 if not self.cloneDestination and len(depotPaths) > 1:
3556 self.cloneDestination = depotPaths[-1]
3557 depotPaths = depotPaths[:-1]
3559 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3560 for p in depotPaths:
3561 if not p.startswith("//"):
3562 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3565 if not self.cloneDestination:
3566 self.cloneDestination = self.defaultDestination(args)
3568 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3570 if not os.path.exists(self.cloneDestination):
3571 os.makedirs(self.cloneDestination)
3572 chdir(self.cloneDestination)
3574 init_cmd = [ "git", "init" ]
3576 init_cmd.append("--bare")
3577 retcode = subprocess.call(init_cmd)
3579 raise CalledProcessError(retcode, init_cmd)
3581 if not P4Sync.run(self, depotPaths):
3584 # create a master branch and check out a work tree
3585 if gitBranchExists(self.branch):
3586 system([ "git", "branch", "master", self.branch ])
3587 if not self.cloneBare:
3588 system([ "git", "checkout", "-f" ])
3590 print 'Not checking out any branch, use ' \
3591 '"git checkout -q -b master <branch>"'
3593 # auto-set this variable if invoked with --use-client-spec
3594 if self.useClientSpec_from_options:
3595 system("git config --bool git-p4.useclientspec true")
3599 class P4Branches(Command):
3601 Command.__init__(self)
3603 self.description = ("Shows the git branches that hold imports and their "
3604 + "corresponding perforce depot paths")
3605 self.verbose = False
3607 def run(self, args):
3608 if originP4BranchesExist():
3609 createOrUpdateBranchesFromOrigin()
3611 cmdline = "git rev-parse --symbolic "
3612 cmdline += " --remotes"
3614 for line in read_pipe_lines(cmdline):
3617 if not line.startswith('p4/') or line == "p4/HEAD":
3621 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3622 settings = extractSettingsGitLog(log)
3624 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3627 class HelpFormatter(optparse.IndentedHelpFormatter):
3629 optparse.IndentedHelpFormatter.__init__(self)
3631 def format_description(self, description):
3633 return description + "\n"
3637 def printUsage(commands):
3638 print "usage: %s <command> [options]" % sys.argv[0]
3640 print "valid commands: %s" % ", ".join(commands)
3642 print "Try %s <command> --help for command specific help." % sys.argv[0]
3647 "submit" : P4Submit,
3648 "commit" : P4Submit,
3650 "rebase" : P4Rebase,
3652 "rollback" : P4RollBack,
3653 "branches" : P4Branches
3658 if len(sys.argv[1:]) == 0:
3659 printUsage(commands.keys())
3662 cmdName = sys.argv[1]
3664 klass = commands[cmdName]
3667 print "unknown command %s" % cmdName
3669 printUsage(commands.keys())
3672 options = cmd.options
3673 cmd.gitdir = os.environ.get("GIT_DIR", None)
3677 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3679 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3681 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3683 description = cmd.description,
3684 formatter = HelpFormatter())
3686 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3688 verbose = cmd.verbose
3690 if cmd.gitdir == None:
3691 cmd.gitdir = os.path.abspath(".git")
3692 if not isValidGitDir(cmd.gitdir):
3693 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3694 if os.path.exists(cmd.gitdir):
3695 cdup = read_pipe("git rev-parse --show-cdup").strip()
3699 if not isValidGitDir(cmd.gitdir):
3700 if isValidGitDir(cmd.gitdir + "/.git"):
3701 cmd.gitdir += "/.git"
3703 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3705 os.environ["GIT_DIR"] = cmd.gitdir
3707 if not cmd.run(args):
3712 if __name__ == '__main__':