3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self, returncode, cmd):
39 self.returncode = returncode
42 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
52 def p4_build_cmd(cmd):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user = gitConfig("git-p4.user")
63 real_cmd += ["-u",user]
65 password = gitConfig("git-p4.password")
67 real_cmd += ["-P", password]
69 port = gitConfig("git-p4.port")
71 real_cmd += ["-p", port]
73 host = gitConfig("git-p4.host")
75 real_cmd += ["-H", host]
77 client = gitConfig("git-p4.client")
79 real_cmd += ["-c", client]
82 if isinstance(cmd,basestring):
83 real_cmd = ' '.join(real_cmd) + ' ' + cmd
88 def chdir(path, is_client_path=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
106 if not is_client_path:
108 os.environ['PWD'] = path
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform.system() == 'Windows':
113 free_bytes = ctypes.c_ulonglong(0)
114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115 return free_bytes.value
117 st = os.statvfs(os.getcwd())
118 return st.f_bavail * st.f_frsize
124 sys.stderr.write(msg + "\n")
127 def write_pipe(c, stdin):
129 sys.stderr.write('Writing pipe: %s\n' % str(c))
131 expand = isinstance(c,basestring)
132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
134 val = pipe.write(stdin)
137 die('Command failed: %s' % str(c))
141 def p4_write_pipe(c, stdin):
142 real_cmd = p4_build_cmd(c)
143 return write_pipe(real_cmd, stdin)
145 def read_pipe(c, ignore_error=False):
147 sys.stderr.write('Reading pipe: %s\n' % str(c))
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151 (out, err) = p.communicate()
152 if p.returncode != 0 and not ignore_error:
153 die('Command failed: %s\nError: %s' % (str(c), err))
156 def p4_read_pipe(c, ignore_error=False):
157 real_cmd = p4_build_cmd(c)
158 return read_pipe(real_cmd, ignore_error)
160 def read_pipe_lines(c):
162 sys.stderr.write('Reading pipe: %s\n' % str(c))
164 expand = isinstance(c, basestring)
165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
167 val = pipe.readlines()
168 if pipe.close() or p.wait():
169 die('Command failed: %s' % str(c))
173 def p4_read_pipe_lines(c):
174 """Specifically invoke p4 on the command supplied. """
175 real_cmd = p4_build_cmd(c)
176 return read_pipe_lines(real_cmd)
178 def p4_has_command(cmd):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd = p4_build_cmd(["help", cmd])
182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183 stderr=subprocess.PIPE)
185 return p.returncode == 0
187 def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
193 if not p4_has_command("move"):
195 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197 (out, err) = p.communicate()
198 # return code will be 1 in either case
199 if err.find("Invalid option") >= 0:
201 if err.find("disabled") >= 0:
203 # assume it failed because @... was invalid changelist
206 def system(cmd, ignore_error=False):
207 expand = isinstance(cmd,basestring)
209 sys.stderr.write("executing %s\n" % str(cmd))
210 retcode = subprocess.call(cmd, shell=expand)
211 if retcode and not ignore_error:
212 raise CalledProcessError(retcode, cmd)
217 """Specifically invoke p4 as the system command. """
218 real_cmd = p4_build_cmd(cmd)
219 expand = isinstance(real_cmd, basestring)
220 retcode = subprocess.call(real_cmd, shell=expand)
222 raise CalledProcessError(retcode, real_cmd)
224 _p4_version_string = None
225 def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
234 global _p4_version_string
235 if not _p4_version_string:
236 a = p4_read_pipe_lines(["-V"])
237 _p4_version_string = a[-1].rstrip()
238 return _p4_version_string
240 def p4_integrate(src, dest):
241 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
243 def p4_sync(f, *options):
244 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
247 # forcibly add file names with wildcards
248 if wildcard_present(f):
249 p4_system(["add", "-f", f])
251 p4_system(["add", f])
254 p4_system(["delete", wildcard_encode(f)])
256 def p4_edit(f, *options):
257 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
260 p4_system(["revert", wildcard_encode(f)])
262 def p4_reopen(type, f):
263 p4_system(["reopen", "-t", type, wildcard_encode(f)])
265 def p4_move(src, dest):
266 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
268 def p4_last_change():
269 results = p4CmdList(["changes", "-m", "1"])
270 return int(results[0]['change'])
272 def p4_describe(change):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
277 ds = p4CmdList(["describe", "-s", str(change)])
279 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
283 if "p4ExitCode" in d:
284 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
287 if d["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
291 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers. See "p4 help filetypes"
298 # for a list and explanation.
300 def split_p4_type(p4type):
302 p4_filetypes_historical = {
303 "ctempobj": "binary+Sw",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
315 "xtempobj": "binary+Swx",
317 "xunicode": "unicode+x",
320 if p4type in p4_filetypes_historical:
321 p4type = p4_filetypes_historical[p4type]
323 s = p4type.split("+")
331 # return the raw p4 type of a file (text, text+ko, etc)
334 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
335 return results[0]['headType']
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
341 def p4_keywords_regexp_for_type(base, type_mods):
342 if base in ("text", "unicode", "binary"):
344 if "ko" in type_mods:
346 elif "k" in type_mods:
347 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
353 (:[^$\n]+)? # possibly an old expansion, followed by...
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
365 def p4_keywords_regexp_for_file(file):
366 if not os.path.exists(file):
369 (type_base, type_mods) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base, type_mods)
372 def setP4ExecBit(file, mode):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
378 if not isModeExec(mode):
379 p4Type = getP4OpenedType(file)
380 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382 if p4Type[-1] == "+":
383 p4Type = p4Type[0:-1]
385 p4_reopen(p4Type, file)
387 def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
390 result = p4_read_pipe(["opened", wildcard_encode(file)])
391 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
393 return match.group(1)
395 die("Could not determine file type for %s (result: '%s')" % (file, result))
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths):
400 if isinstance(depotPaths,basestring):
401 depotPaths = [depotPaths]
403 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
409 # Return the set of all git tags
412 for line in read_pipe_lines(["git", "tag"]):
417 def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
424 def parseDiffTreeEntry(entry):
425 """Parses a single diff tree entry into its component elements.
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
441 If the pattern is not matched, None is returned."""
443 match = diffTreePattern().next().match(entry)
446 'src_mode': match.group(1),
447 'dst_mode': match.group(2),
448 'src_sha1': match.group(3),
449 'dst_sha1': match.group(4),
450 'status': match.group(5),
451 'status_score': match.group(6),
452 'src': match.group(7),
453 'dst': match.group(10)
457 def isModeExec(mode):
458 # Returns True if the given git mode represents an executable file,
460 return mode[-3:] == "755"
462 def isModeExecChanged(src_mode, dst_mode):
463 return isModeExec(src_mode) != isModeExec(dst_mode)
465 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
467 if isinstance(cmd,basestring):
474 cmd = p4_build_cmd(cmd)
476 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
482 if stdin is not None:
483 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
484 if isinstance(stdin,basestring):
485 stdin_file.write(stdin)
488 stdin_file.write(i + '\n')
492 p4 = subprocess.Popen(cmd,
495 stdout=subprocess.PIPE)
500 entry = marshal.load(p4.stdout)
510 entry["p4ExitCode"] = exitCode
516 list = p4CmdList(cmd)
522 def p4Where(depotPath):
523 if not depotPath.endswith("/"):
525 depotPathLong = depotPath + "..."
526 outputList = p4CmdList(["where", depotPathLong])
528 for entry in outputList:
529 if "depotFile" in entry:
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
535 elif "data" in entry:
536 data = entry.get("data")
537 space = data.find(" ")
538 if data[:space] == depotPath:
543 if output["code"] == "error":
547 clientPath = output.get("path")
548 elif "data" in output:
549 data = output.get("data")
550 lastSpace = data.rfind(" ")
551 clientPath = data[lastSpace + 1:]
553 if clientPath.endswith("..."):
554 clientPath = clientPath[:-3]
557 def currentGitBranch():
558 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
565 def isValidGitDir(path):
566 if (os.path.exists(path + "/HEAD")
567 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
571 def parseRevision(ref):
572 return read_pipe("git rev-parse %s" % ref).strip()
574 def branchExists(ref):
575 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
579 def extractLogMessageFromGitCommit(commit):
582 ## fixme: title is first line of commit, not 1st paragraph.
584 for log in read_pipe_lines("git cat-file commit %s" % commit):
593 def extractSettingsGitLog(log):
595 for line in log.split("\n"):
597 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
601 assignments = m.group(1).split (':')
602 for a in assignments:
604 key = vals[0].strip()
605 val = ('='.join (vals[1:])).strip()
606 if val.endswith ('\"') and val.startswith('"'):
611 paths = values.get("depot-paths")
613 paths = values.get("depot-path")
615 values['depot-paths'] = paths.split(',')
618 def gitBranchExists(branch):
619 proc = subprocess.Popen(["git", "rev-parse", branch],
620 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
621 return proc.wait() == 0;
625 def gitConfig(key, typeSpecifier=None):
626 if not _gitConfig.has_key(key):
627 cmd = [ "git", "config" ]
629 cmd += [ typeSpecifier ]
631 s = read_pipe(cmd, ignore_error=True)
632 _gitConfig[key] = s.strip()
633 return _gitConfig[key]
635 def gitConfigBool(key):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
640 if not _gitConfig.has_key(key):
641 _gitConfig[key] = gitConfig(key, '--bool') == "true"
642 return _gitConfig[key]
644 def gitConfigInt(key):
645 if not _gitConfig.has_key(key):
646 cmd = [ "git", "config", "--int", key ]
647 s = read_pipe(cmd, ignore_error=True)
650 _gitConfig[key] = int(gitConfig(key, '--int'))
652 _gitConfig[key] = None
653 return _gitConfig[key]
655 def gitConfigList(key):
656 if not _gitConfig.has_key(key):
657 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658 _gitConfig[key] = s.strip().split(os.linesep)
659 if _gitConfig[key] == ['']:
661 return _gitConfig[key]
663 def p4BranchesInGit(branchesAreInRemotes=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
672 cmdline = "git rev-parse --symbolic "
673 if branchesAreInRemotes:
674 cmdline += "--remotes"
676 cmdline += "--branches"
678 for line in read_pipe_lines(cmdline):
682 if not line.startswith('p4/'):
684 # special symbolic ref to p4/master
685 if line == "p4/HEAD":
688 # strip off p4/ prefix
689 branch = line[len("p4/"):]
691 branches[branch] = parseRevision(line)
695 def branch_exists(branch):
696 """Make sure that the given ref name really exists."""
698 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700 out, _ = p.communicate()
703 # expect exactly one line of output: the branch name
704 return out.rstrip() == branch
706 def findUpstreamBranchPoint(head = "HEAD"):
707 branches = p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath = {}
710 for branch in branches.keys():
711 tip = branches[branch]
712 log = extractLogMessageFromGitCommit(tip)
713 settings = extractSettingsGitLog(log)
714 if settings.has_key("depot-paths"):
715 paths = ",".join(settings["depot-paths"])
716 branchByDepotPath[paths] = "remotes/p4/" + branch
720 while parent < 65535:
721 commit = head + "~%s" % parent
722 log = extractLogMessageFromGitCommit(commit)
723 settings = extractSettingsGitLog(log)
724 if settings.has_key("depot-paths"):
725 paths = ",".join(settings["depot-paths"])
726 if branchByDepotPath.has_key(paths):
727 return [branchByDepotPath[paths], settings]
731 return ["", settings]
733 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
738 originPrefix = "origin/p4/"
740 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
742 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
745 headName = line[len(originPrefix):]
746 remoteHead = localRefPrefix + headName
749 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750 if (not original.has_key('depot-paths')
751 or not original.has_key('change')):
755 if not gitBranchExists(remoteHead):
757 print "creating %s" % remoteHead
760 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761 if settings.has_key('change') > 0:
762 if settings['depot-paths'] == original['depot-paths']:
763 originP4Change = int(original['change'])
764 p4Change = int(settings['change'])
765 if originP4Change > p4Change:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead, originP4Change,
769 remoteHead, p4Change))
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead, ','.join(original['depot-paths']),
775 remoteHead, ','.join(settings['depot-paths'])))
778 system("git update-ref %s %s" % (remoteHead, originHead))
780 def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
784 def p4ParseNumericChangeRange(parts):
785 changeStart = int(parts[0][1:])
786 if parts[1] == '#head':
787 changeEnd = p4_last_change()
789 changeEnd = int(parts[1])
791 return (changeStart, changeEnd)
793 def chooseBlockSize(blockSize):
797 return defaultBlockSize
799 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
808 if changeRange is None or changeRange == '':
810 changeEnd = p4_last_change()
811 block_size = chooseBlockSize(requestedBlockSize)
813 parts = changeRange.split(',')
814 assert len(parts) == 2
816 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817 block_size = chooseBlockSize(requestedBlockSize)
819 changeStart = parts[0][1:]
821 if requestedBlockSize:
822 die("cannot use --changes-block-size with non-numeric revisions")
827 # Retrieve changes a block at a time, to prevent running
828 # into a MaxResults/MaxScanRows error from the server.
834 end = min(changeEnd, changeStart + block_size)
835 revisionRange = "%d,%d" % (changeStart, end)
837 revisionRange = "%s,%s" % (changeStart, changeEnd)
840 cmd += ["%s...@%s" % (p, revisionRange)]
842 # Insert changes in chronological order
843 for line in reversed(p4_read_pipe_lines(cmd)):
844 changes.append(int(line.split(" ")[1]))
852 changeStart = end + 1
854 changes = sorted(changes)
857 def p4PathStartsWith(path, prefix):
858 # This method tries to remedy a potential mixed-case issue:
860 # If UserA adds //depot/DirA/file1
861 # and UserB adds //depot/dira/file2
863 # we may or may not have a problem. If you have core.ignorecase=true,
864 # we treat DirA and dira as the same directory
865 if gitConfigBool("core.ignorecase"):
866 return path.lower().startswith(prefix.lower())
867 return path.startswith(prefix)
870 """Look at the p4 client spec, create a View() object that contains
871 all the mappings, and return it."""
873 specList = p4CmdList("client -o")
874 if len(specList) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' %
878 # dictionary of all client parameters
882 client_name = entry["Client"]
884 # just the keys that start with "View"
885 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
888 view = View(client_name)
890 # append the lines, in order, to the view
891 for view_num in range(len(view_keys)):
892 k = "View%d" % view_num
893 if k not in view_keys:
894 die("Expected view key %s missing" % k)
895 view.append(entry[k])
900 """Grab the client directory."""
902 output = p4CmdList("client -o")
904 die('Output from "client -o" is %d lines, expecting 1' % len(output))
907 if "Root" not in entry:
908 die('Client has no "Root"')
913 # P4 wildcards are not allowed in filenames. P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
917 def wildcard_decode(path):
918 # Search for and fix just these four characters. Do % last so
919 # that fixing it does not inadvertently create new %-escapes.
920 # Cannot have * in a filename in windows; untested as to
921 # what p4 would do in such a case.
922 if not platform.system() == "Windows":
923 path = path.replace("%2A", "*")
924 path = path.replace("%23", "#") \
925 .replace("%40", "@") \
929 def wildcard_encode(path):
930 # do % first to avoid double-encoding the %s introduced here
931 path = path.replace("%", "%25") \
932 .replace("*", "%2A") \
933 .replace("#", "%23") \
937 def wildcard_present(path):
938 m = re.search("[*#@%]", path)
941 class LargeFileSystem(object):
942 """Base class for large file system support."""
944 def __init__(self, writeToGitStream):
945 self.largeFiles = set()
946 self.writeToGitStream = writeToGitStream
948 def generatePointer(self, cloneDestination, contentFile):
949 """Return the content of a pointer file that is stored in Git instead of
950 the actual content."""
951 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
953 def pushFile(self, localLargeFile):
954 """Push the actual content which is not stored in the Git repository to
956 assert False, "Method 'pushFile' required in " + self.__class__.__name__
958 def hasLargeFileExtension(self, relPath):
961 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
965 def generateTempFile(self, contents):
966 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
970 return contentFile.name
972 def exceedsLargeFileThreshold(self, relPath, contents):
973 if gitConfigInt('git-p4.largeFileThreshold'):
974 contentsSize = sum(len(d) for d in contents)
975 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
977 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978 contentsSize = sum(len(d) for d in contents)
979 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
981 contentTempFile = self.generateTempFile(contents)
982 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
986 compressedContentsSize = zf.infolist()[0].compress_size
987 os.remove(contentTempFile)
988 os.remove(compressedContentFile.name)
989 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
993 def addLargeFile(self, relPath):
994 self.largeFiles.add(relPath)
996 def removeLargeFile(self, relPath):
997 self.largeFiles.remove(relPath)
999 def isLargeFile(self, relPath):
1000 return relPath in self.largeFiles
1002 def processContent(self, git_mode, relPath, contents):
1003 """Processes the content of git fast import. This method decides if a
1004 file is stored in the large file system and handles all necessary
1006 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007 contentTempFile = self.generateTempFile(contents)
1008 (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1010 # Move temp file to final location in large file system
1011 largeFileDir = os.path.dirname(localLargeFile)
1012 if not os.path.isdir(largeFileDir):
1013 os.makedirs(largeFileDir)
1014 shutil.move(contentTempFile, localLargeFile)
1015 self.addLargeFile(relPath)
1016 if gitConfigBool('git-p4.largeFilePush'):
1017 self.pushFile(localLargeFile)
1019 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020 return (git_mode, contents)
1022 class MockLFS(LargeFileSystem):
1023 """Mock large file system for testing."""
1025 def generatePointer(self, contentFile):
1026 """The pointer content is the original content prefixed with "pointer-".
1027 The local filename of the large file storage is derived from the file content.
1029 with open(contentFile, 'r') as f:
1032 pointerContents = 'pointer-' + content
1033 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034 return (gitMode, pointerContents, localLargeFile)
1036 def pushFile(self, localLargeFile):
1037 """The remote filename of the large file storage is the same as the local
1038 one but in a different directory.
1040 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041 if not os.path.exists(remotePath):
1042 os.makedirs(remotePath)
1043 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1045 class GitLFS(LargeFileSystem):
1046 """Git LFS as backend for the git-p4 large file system.
1047 See https://git-lfs.github.com/ for details."""
1049 def __init__(self, *args):
1050 LargeFileSystem.__init__(self, *args)
1051 self.baseGitAttributes = []
1053 def generatePointer(self, contentFile):
1054 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055 mode and content which is stored in the Git repository instead of
1056 the actual content. Return also the new location of the actual
1059 pointerProcess = subprocess.Popen(
1060 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061 stdout=subprocess.PIPE
1063 pointerFile = pointerProcess.stdout.read()
1064 if pointerProcess.wait():
1065 os.remove(contentFile)
1066 die('git-lfs pointer command failed. Did you install the extension?')
1068 # Git LFS removed the preamble in the output of the 'pointer' command
1069 # starting from version 1.2.0. Check for the preamble here to support
1071 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1072 if pointerFile.startswith('Git LFS pointer for'):
1073 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1075 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1076 localLargeFile = os.path.join(
1078 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1081 # LFS Spec states that pointer files should not have the executable bit set.
1083 return (gitMode, pointerFile, localLargeFile)
1085 def pushFile(self, localLargeFile):
1086 uploadProcess = subprocess.Popen(
1087 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1089 if uploadProcess.wait():
1090 die('git-lfs push command failed. Did you define a remote?')
1092 def generateGitAttributes(self):
1094 self.baseGitAttributes +
1098 '# Git LFS (see https://git-lfs.github.com/)\n',
1101 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1102 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1104 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1105 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1109 def addLargeFile(self, relPath):
1110 LargeFileSystem.addLargeFile(self, relPath)
1111 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1113 def removeLargeFile(self, relPath):
1114 LargeFileSystem.removeLargeFile(self, relPath)
1115 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1117 def processContent(self, git_mode, relPath, contents):
1118 if relPath == '.gitattributes':
1119 self.baseGitAttributes = contents
1120 return (git_mode, self.generateGitAttributes())
1122 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1126 self.usage = "usage: %prog [options]"
1127 self.needsGit = True
1128 self.verbose = False
1132 self.userMapFromPerforceServer = False
1133 self.myP4UserId = None
1137 return self.myP4UserId
1139 results = p4CmdList("user -o")
1141 if r.has_key('User'):
1142 self.myP4UserId = r['User']
1144 die("Could not find your p4 user id")
1146 def p4UserIsMe(self, p4User):
1147 # return True if the given p4 user is actually me
1148 me = self.p4UserId()
1149 if not p4User or p4User != me:
1154 def getUserCacheFilename(self):
1155 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1156 return home + "/.gitp4-usercache.txt"
1158 def getUserMapFromPerforceServer(self):
1159 if self.userMapFromPerforceServer:
1164 for output in p4CmdList("users"):
1165 if not output.has_key("User"):
1167 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1168 self.emails[output["Email"]] = output["User"]
1170 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1171 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1172 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1173 if mapUser and len(mapUser[0]) == 3:
1174 user = mapUser[0][0]
1175 fullname = mapUser[0][1]
1176 email = mapUser[0][2]
1177 self.users[user] = fullname + " <" + email + ">"
1178 self.emails[email] = user
1181 for (key, val) in self.users.items():
1182 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1184 open(self.getUserCacheFilename(), "wb").write(s)
1185 self.userMapFromPerforceServer = True
1187 def loadUserMapFromCache(self):
1189 self.userMapFromPerforceServer = False
1191 cache = open(self.getUserCacheFilename(), "rb")
1192 lines = cache.readlines()
1195 entry = line.strip().split("\t")
1196 self.users[entry[0]] = entry[1]
1198 self.getUserMapFromPerforceServer()
1200 class P4Debug(Command):
1202 Command.__init__(self)
1204 self.description = "A tool to debug the output of p4 -G."
1205 self.needsGit = False
1207 def run(self, args):
1209 for output in p4CmdList(args):
1210 print 'Element: %d' % j
1215 class P4RollBack(Command):
1217 Command.__init__(self)
1219 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1221 self.description = "A tool to debug the multi-branch import. Don't use :)"
1222 self.rollbackLocalBranches = False
1224 def run(self, args):
1227 maxChange = int(args[0])
1229 if "p4ExitCode" in p4Cmd("changes -m 1"):
1230 die("Problems executing p4");
1232 if self.rollbackLocalBranches:
1233 refPrefix = "refs/heads/"
1234 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1236 refPrefix = "refs/remotes/"
1237 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1240 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1242 ref = refPrefix + line
1243 log = extractLogMessageFromGitCommit(ref)
1244 settings = extractSettingsGitLog(log)
1246 depotPaths = settings['depot-paths']
1247 change = settings['change']
1251 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1252 for p in depotPaths]))) == 0:
1253 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1254 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1257 while change and int(change) > maxChange:
1260 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1261 system("git update-ref %s \"%s^\"" % (ref, ref))
1262 log = extractLogMessageFromGitCommit(ref)
1263 settings = extractSettingsGitLog(log)
1266 depotPaths = settings['depot-paths']
1267 change = settings['change']
1270 print "%s rewound to %s" % (ref, change)
1274 class P4Submit(Command, P4UserMap):
1276 conflict_behavior_choices = ("ask", "skip", "quit")
1279 Command.__init__(self)
1280 P4UserMap.__init__(self)
1282 optparse.make_option("--origin", dest="origin"),
1283 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1284 # preserve the user, requires relevant p4 permissions
1285 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1286 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1287 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1288 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1289 optparse.make_option("--conflict", dest="conflict_behavior",
1290 choices=self.conflict_behavior_choices),
1291 optparse.make_option("--branch", dest="branch"),
1293 self.description = "Submit changes from git to the perforce depot."
1294 self.usage += " [name of git branch to submit into perforce depot]"
1296 self.detectRenames = False
1297 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1298 self.dry_run = False
1299 self.prepare_p4_only = False
1300 self.conflict_behavior = None
1301 self.isWindows = (platform.system() == "Windows")
1302 self.exportLabels = False
1303 self.p4HasMoveCommand = p4_has_move_command()
1306 if gitConfig('git-p4.largeFileSystem'):
1307 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1310 if len(p4CmdList("opened ...")) > 0:
1311 die("You have files opened with perforce! Close them before starting the sync.")
1313 def separate_jobs_from_description(self, message):
1314 """Extract and return a possible Jobs field in the commit
1315 message. It goes into a separate section in the p4 change
1318 A jobs line starts with "Jobs:" and looks like a new field
1319 in a form. Values are white-space separated on the same
1320 line or on following lines that start with a tab.
1322 This does not parse and extract the full git commit message
1323 like a p4 form. It just sees the Jobs: line as a marker
1324 to pass everything from then on directly into the p4 form,
1325 but outside the description section.
1327 Return a tuple (stripped log message, jobs string)."""
1329 m = re.search(r'^Jobs:', message, re.MULTILINE)
1331 return (message, None)
1333 jobtext = message[m.start():]
1334 stripped_message = message[:m.start()].rstrip()
1335 return (stripped_message, jobtext)
1337 def prepareLogMessage(self, template, message, jobs):
1338 """Edits the template returned from "p4 change -o" to insert
1339 the message in the Description field, and the jobs text in
1343 inDescriptionSection = False
1345 for line in template.split("\n"):
1346 if line.startswith("#"):
1347 result += line + "\n"
1350 if inDescriptionSection:
1351 if line.startswith("Files:") or line.startswith("Jobs:"):
1352 inDescriptionSection = False
1353 # insert Jobs section
1355 result += jobs + "\n"
1359 if line.startswith("Description:"):
1360 inDescriptionSection = True
1362 for messageLine in message.split("\n"):
1363 line += "\t" + messageLine + "\n"
1365 result += line + "\n"
1369 def patchRCSKeywords(self, file, pattern):
1370 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1371 (handle, outFileName) = tempfile.mkstemp(dir='.')
1373 outFile = os.fdopen(handle, "w+")
1374 inFile = open(file, "r")
1375 regexp = re.compile(pattern, re.VERBOSE)
1376 for line in inFile.readlines():
1377 line = regexp.sub(r'$\1$', line)
1381 # Forcibly overwrite the original file
1383 shutil.move(outFileName, file)
1385 # cleanup our temporary file
1386 os.unlink(outFileName)
1387 print "Failed to strip RCS keywords in %s" % file
1390 print "Patched up RCS keywords in %s" % file
1392 def p4UserForCommit(self,id):
1393 # Return the tuple (perforce user,git email) for a given git commit id
1394 self.getUserMapFromPerforceServer()
1395 gitEmail = read_pipe(["git", "log", "--max-count=1",
1396 "--format=%ae", id])
1397 gitEmail = gitEmail.strip()
1398 if not self.emails.has_key(gitEmail):
1399 return (None,gitEmail)
1401 return (self.emails[gitEmail],gitEmail)
1403 def checkValidP4Users(self,commits):
1404 # check if any git authors cannot be mapped to p4 users
1406 (user,email) = self.p4UserForCommit(id)
1408 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1409 if gitConfigBool("git-p4.allowMissingP4Users"):
1412 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1414 def lastP4Changelist(self):
1415 # Get back the last changelist number submitted in this client spec. This
1416 # then gets used to patch up the username in the change. If the same
1417 # client spec is being used by multiple processes then this might go
1419 results = p4CmdList("client -o") # find the current client
1422 if r.has_key('Client'):
1423 client = r['Client']
1426 die("could not get client spec")
1427 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1429 if r.has_key('change'):
1431 die("Could not get changelist number for last submit - cannot patch up user details")
1433 def modifyChangelistUser(self, changelist, newUser):
1434 # fixup the user field of a changelist after it has been submitted.
1435 changes = p4CmdList("change -o %s" % changelist)
1436 if len(changes) != 1:
1437 die("Bad output from p4 change modifying %s to user %s" %
1438 (changelist, newUser))
1441 if c['User'] == newUser: return # nothing to do
1443 input = marshal.dumps(c)
1445 result = p4CmdList("change -f -i", stdin=input)
1447 if r.has_key('code'):
1448 if r['code'] == 'error':
1449 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1450 if r.has_key('data'):
1451 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1453 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1455 def canChangeChangelists(self):
1456 # check to see if we have p4 admin or super-user permissions, either of
1457 # which are required to modify changelists.
1458 results = p4CmdList(["protects", self.depotPath])
1460 if r.has_key('perm'):
1461 if r['perm'] == 'admin':
1463 if r['perm'] == 'super':
1467 def prepareSubmitTemplate(self):
1468 """Run "p4 change -o" to grab a change specification template.
1469 This does not use "p4 -G", as it is nice to keep the submission
1470 template in original order, since a human might edit it.
1472 Remove lines in the Files section that show changes to files
1473 outside the depot path we're committing into."""
1475 [upstream, settings] = findUpstreamBranchPoint()
1478 inFilesSection = False
1479 for line in p4_read_pipe_lines(['change', '-o']):
1480 if line.endswith("\r\n"):
1481 line = line[:-2] + "\n"
1483 if line.startswith("\t"):
1484 # path starts and ends with a tab
1486 lastTab = path.rfind("\t")
1488 path = path[:lastTab]
1489 if settings.has_key('depot-paths'):
1490 if not [p for p in settings['depot-paths']
1491 if p4PathStartsWith(path, p)]:
1494 if not p4PathStartsWith(path, self.depotPath):
1497 inFilesSection = False
1499 if line.startswith("Files:"):
1500 inFilesSection = True
1506 def edit_template(self, template_file):
1507 """Invoke the editor to let the user change the submission
1508 message. Return true if okay to continue with the submit."""
1510 # if configured to skip the editing part, just submit
1511 if gitConfigBool("git-p4.skipSubmitEdit"):
1514 # look at the modification time, to check later if the user saved
1516 mtime = os.stat(template_file).st_mtime
1519 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1520 editor = os.environ.get("P4EDITOR")
1522 editor = read_pipe("git var GIT_EDITOR").strip()
1523 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1525 # If the file was not saved, prompt to see if this patch should
1526 # be skipped. But skip this verification step if configured so.
1527 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1530 # modification time updated means user saved the file
1531 if os.stat(template_file).st_mtime > mtime:
1535 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1541 def get_diff_description(self, editedFiles, filesToAdd):
1543 if os.environ.has_key("P4DIFF"):
1544 del(os.environ["P4DIFF"])
1546 for editedFile in editedFiles:
1547 diff += p4_read_pipe(['diff', '-du',
1548 wildcard_encode(editedFile)])
1552 for newFile in filesToAdd:
1553 newdiff += "==== new file ====\n"
1554 newdiff += "--- /dev/null\n"
1555 newdiff += "+++ %s\n" % newFile
1556 f = open(newFile, "r")
1557 for line in f.readlines():
1558 newdiff += "+" + line
1561 return (diff + newdiff).replace('\r\n', '\n')
1563 def applyCommit(self, id):
1564 """Apply one commit, return True if it succeeded."""
1566 print "Applying", read_pipe(["git", "show", "-s",
1567 "--format=format:%h %s", id])
1569 (p4User, gitEmail) = self.p4UserForCommit(id)
1571 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1573 filesToChangeType = set()
1574 filesToDelete = set()
1576 pureRenameCopy = set()
1577 filesToChangeExecBit = {}
1580 diff = parseDiffTreeEntry(line)
1581 modifier = diff['status']
1585 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1586 filesToChangeExecBit[path] = diff['dst_mode']
1587 editedFiles.add(path)
1588 elif modifier == "A":
1589 filesToAdd.add(path)
1590 filesToChangeExecBit[path] = diff['dst_mode']
1591 if path in filesToDelete:
1592 filesToDelete.remove(path)
1593 elif modifier == "D":
1594 filesToDelete.add(path)
1595 if path in filesToAdd:
1596 filesToAdd.remove(path)
1597 elif modifier == "C":
1598 src, dest = diff['src'], diff['dst']
1599 p4_integrate(src, dest)
1600 pureRenameCopy.add(dest)
1601 if diff['src_sha1'] != diff['dst_sha1']:
1603 pureRenameCopy.discard(dest)
1604 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1606 pureRenameCopy.discard(dest)
1607 filesToChangeExecBit[dest] = diff['dst_mode']
1609 # turn off read-only attribute
1610 os.chmod(dest, stat.S_IWRITE)
1612 editedFiles.add(dest)
1613 elif modifier == "R":
1614 src, dest = diff['src'], diff['dst']
1615 if self.p4HasMoveCommand:
1616 p4_edit(src) # src must be open before move
1617 p4_move(src, dest) # opens for (move/delete, move/add)
1619 p4_integrate(src, dest)
1620 if diff['src_sha1'] != diff['dst_sha1']:
1623 pureRenameCopy.add(dest)
1624 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1625 if not self.p4HasMoveCommand:
1626 p4_edit(dest) # with move: already open, writable
1627 filesToChangeExecBit[dest] = diff['dst_mode']
1628 if not self.p4HasMoveCommand:
1630 os.chmod(dest, stat.S_IWRITE)
1632 filesToDelete.add(src)
1633 editedFiles.add(dest)
1634 elif modifier == "T":
1635 filesToChangeType.add(path)
1637 die("unknown modifier %s for %s" % (modifier, path))
1639 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1640 patchcmd = diffcmd + " | git apply "
1641 tryPatchCmd = patchcmd + "--check -"
1642 applyPatchCmd = patchcmd + "--check --apply -"
1643 patch_succeeded = True
1645 if os.system(tryPatchCmd) != 0:
1646 fixed_rcs_keywords = False
1647 patch_succeeded = False
1648 print "Unfortunately applying the change failed!"
1650 # Patch failed, maybe it's just RCS keyword woes. Look through
1651 # the patch to see if that's possible.
1652 if gitConfigBool("git-p4.attemptRCSCleanup"):
1656 for file in editedFiles | filesToDelete:
1657 # did this file's delta contain RCS keywords?
1658 pattern = p4_keywords_regexp_for_file(file)
1661 # this file is a possibility...look for RCS keywords.
1662 regexp = re.compile(pattern, re.VERBOSE)
1663 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1664 if regexp.search(line):
1666 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1667 kwfiles[file] = pattern
1670 for file in kwfiles:
1672 print "zapping %s with %s" % (line,pattern)
1673 # File is being deleted, so not open in p4. Must
1674 # disable the read-only bit on windows.
1675 if self.isWindows and file not in editedFiles:
1676 os.chmod(file, stat.S_IWRITE)
1677 self.patchRCSKeywords(file, kwfiles[file])
1678 fixed_rcs_keywords = True
1680 if fixed_rcs_keywords:
1681 print "Retrying the patch with RCS keywords cleaned up"
1682 if os.system(tryPatchCmd) == 0:
1683 patch_succeeded = True
1685 if not patch_succeeded:
1686 for f in editedFiles:
1691 # Apply the patch for real, and do add/delete/+x handling.
1693 system(applyPatchCmd)
1695 for f in filesToChangeType:
1696 p4_edit(f, "-t", "auto")
1697 for f in filesToAdd:
1699 for f in filesToDelete:
1703 # Set/clear executable bits
1704 for f in filesToChangeExecBit.keys():
1705 mode = filesToChangeExecBit[f]
1706 setP4ExecBit(f, mode)
1709 # Build p4 change description, starting with the contents
1710 # of the git commit message.
1712 logMessage = extractLogMessageFromGitCommit(id)
1713 logMessage = logMessage.strip()
1714 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1716 template = self.prepareSubmitTemplate()
1717 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1719 if self.preserveUser:
1720 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1722 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1723 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1724 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1725 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1727 separatorLine = "######## everything below this line is just the diff #######\n"
1728 if not self.prepare_p4_only:
1729 submitTemplate += separatorLine
1730 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1732 (handle, fileName) = tempfile.mkstemp()
1733 tmpFile = os.fdopen(handle, "w+b")
1735 submitTemplate = submitTemplate.replace("\n", "\r\n")
1736 tmpFile.write(submitTemplate)
1739 if self.prepare_p4_only:
1741 # Leave the p4 tree prepared, and the submit template around
1742 # and let the user decide what to do next
1745 print "P4 workspace prepared for submission."
1746 print "To submit or revert, go to client workspace"
1747 print " " + self.clientPath
1749 print "To submit, use \"p4 submit\" to write a new description,"
1750 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1751 " \"git p4\"." % fileName
1752 print "You can delete the file \"%s\" when finished." % fileName
1754 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1755 print "To preserve change ownership by user %s, you must\n" \
1756 "do \"p4 change -f <change>\" after submitting and\n" \
1757 "edit the User field."
1759 print "After submitting, renamed files must be re-synced."
1760 print "Invoke \"p4 sync -f\" on each of these files:"
1761 for f in pureRenameCopy:
1765 print "To revert the changes, use \"p4 revert ...\", and delete"
1766 print "the submit template file \"%s\"" % fileName
1768 print "Since the commit adds new files, they must be deleted:"
1769 for f in filesToAdd:
1775 # Let the user edit the change description, then submit it.
1780 if self.edit_template(fileName):
1781 # read the edited message and submit
1782 tmpFile = open(fileName, "rb")
1783 message = tmpFile.read()
1786 message = message.replace("\r\n", "\n")
1787 submitTemplate = message[:message.index(separatorLine)]
1788 p4_write_pipe(['submit', '-i'], submitTemplate)
1790 if self.preserveUser:
1792 # Get last changelist number. Cannot easily get it from
1793 # the submit command output as the output is
1795 changelist = self.lastP4Changelist()
1796 self.modifyChangelistUser(changelist, p4User)
1798 # The rename/copy happened by applying a patch that created a
1799 # new file. This leaves it writable, which confuses p4.
1800 for f in pureRenameCopy:
1807 print "Submission cancelled, undoing p4 changes."
1808 for f in editedFiles:
1810 for f in filesToAdd:
1813 for f in filesToDelete:
1819 # Export git tags as p4 labels. Create a p4 label and then tag
1821 def exportGitTags(self, gitTags):
1822 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1823 if len(validLabelRegexp) == 0:
1824 validLabelRegexp = defaultLabelRegexp
1825 m = re.compile(validLabelRegexp)
1827 for name in gitTags:
1829 if not m.match(name):
1831 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1834 # Get the p4 commit this corresponds to
1835 logMessage = extractLogMessageFromGitCommit(name)
1836 values = extractSettingsGitLog(logMessage)
1838 if not values.has_key('change'):
1839 # a tag pointing to something not sent to p4; ignore
1841 print "git tag %s does not give a p4 commit" % name
1844 changelist = values['change']
1846 # Get the tag details.
1850 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1853 if re.match(r'tag\s+', l):
1855 elif re.match(r'\s*$', l):
1862 body = ["lightweight tag imported by git p4\n"]
1864 # Create the label - use the same view as the client spec we are using
1865 clientSpec = getClientSpec()
1867 labelTemplate = "Label: %s\n" % name
1868 labelTemplate += "Description:\n"
1870 labelTemplate += "\t" + b + "\n"
1871 labelTemplate += "View:\n"
1872 for depot_side in clientSpec.mappings:
1873 labelTemplate += "\t%s\n" % depot_side
1876 print "Would create p4 label %s for tag" % name
1877 elif self.prepare_p4_only:
1878 print "Not creating p4 label %s for tag due to option" \
1879 " --prepare-p4-only" % name
1881 p4_write_pipe(["label", "-i"], labelTemplate)
1884 p4_system(["tag", "-l", name] +
1885 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1888 print "created p4 label for tag %s" % name
1890 def run(self, args):
1892 self.master = currentGitBranch()
1893 elif len(args) == 1:
1894 self.master = args[0]
1895 if not branchExists(self.master):
1896 die("Branch %s does not exist" % self.master)
1901 allowSubmit = gitConfig("git-p4.allowSubmit")
1902 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1903 die("%s is not in git-p4.allowSubmit" % self.master)
1905 [upstream, settings] = findUpstreamBranchPoint()
1906 self.depotPath = settings['depot-paths'][0]
1907 if len(self.origin) == 0:
1908 self.origin = upstream
1910 if self.preserveUser:
1911 if not self.canChangeChangelists():
1912 die("Cannot preserve user names without p4 super-user or admin permissions")
1914 # if not set from the command line, try the config file
1915 if self.conflict_behavior is None:
1916 val = gitConfig("git-p4.conflict")
1918 if val not in self.conflict_behavior_choices:
1919 die("Invalid value '%s' for config git-p4.conflict" % val)
1922 self.conflict_behavior = val
1925 print "Origin branch is " + self.origin
1927 if len(self.depotPath) == 0:
1928 print "Internal error: cannot locate perforce depot path from existing branches"
1931 self.useClientSpec = False
1932 if gitConfigBool("git-p4.useclientspec"):
1933 self.useClientSpec = True
1934 if self.useClientSpec:
1935 self.clientSpecDirs = getClientSpec()
1937 # Check for the existence of P4 branches
1938 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1940 if self.useClientSpec and not branchesDetected:
1941 # all files are relative to the client spec
1942 self.clientPath = getClientRoot()
1944 self.clientPath = p4Where(self.depotPath)
1946 if self.clientPath == "":
1947 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1949 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1950 self.oldWorkingDirectory = os.getcwd()
1952 # ensure the clientPath exists
1953 new_client_dir = False
1954 if not os.path.exists(self.clientPath):
1955 new_client_dir = True
1956 os.makedirs(self.clientPath)
1958 chdir(self.clientPath, is_client_path=True)
1960 print "Would synchronize p4 checkout in %s" % self.clientPath
1962 print "Synchronizing p4 checkout..."
1964 # old one was destroyed, and maybe nobody told p4
1965 p4_sync("...", "-f")
1972 commitish = self.master
1976 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1977 commits.append(line.strip())
1980 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1981 self.checkAuthorship = False
1983 self.checkAuthorship = True
1985 if self.preserveUser:
1986 self.checkValidP4Users(commits)
1989 # Build up a set of options to be passed to diff when
1990 # submitting each commit to p4.
1992 if self.detectRenames:
1993 # command-line -M arg
1994 self.diffOpts = "-M"
1996 # If not explicitly set check the config variable
1997 detectRenames = gitConfig("git-p4.detectRenames")
1999 if detectRenames.lower() == "false" or detectRenames == "":
2001 elif detectRenames.lower() == "true":
2002 self.diffOpts = "-M"
2004 self.diffOpts = "-M%s" % detectRenames
2006 # no command-line arg for -C or --find-copies-harder, just
2008 detectCopies = gitConfig("git-p4.detectCopies")
2009 if detectCopies.lower() == "false" or detectCopies == "":
2011 elif detectCopies.lower() == "true":
2012 self.diffOpts += " -C"
2014 self.diffOpts += " -C%s" % detectCopies
2016 if gitConfigBool("git-p4.detectCopiesHarder"):
2017 self.diffOpts += " --find-copies-harder"
2020 # Apply the commits, one at a time. On failure, ask if should
2021 # continue to try the rest of the patches, or quit.
2026 last = len(commits) - 1
2027 for i, commit in enumerate(commits):
2029 print " ", read_pipe(["git", "show", "-s",
2030 "--format=format:%h %s", commit])
2033 ok = self.applyCommit(commit)
2035 applied.append(commit)
2037 if self.prepare_p4_only and i < last:
2038 print "Processing only the first commit due to option" \
2039 " --prepare-p4-only"
2044 # prompt for what to do, or use the option/variable
2045 if self.conflict_behavior == "ask":
2046 print "What do you want to do?"
2047 response = raw_input("[s]kip this commit but apply"
2048 " the rest, or [q]uit? ")
2051 elif self.conflict_behavior == "skip":
2053 elif self.conflict_behavior == "quit":
2056 die("Unknown conflict_behavior '%s'" %
2057 self.conflict_behavior)
2059 if response[0] == "s":
2060 print "Skipping this commit, but applying the rest"
2062 if response[0] == "q":
2069 chdir(self.oldWorkingDirectory)
2073 elif self.prepare_p4_only:
2075 elif len(commits) == len(applied):
2076 print "All commits applied!"
2080 sync.branch = self.branch
2087 if len(applied) == 0:
2088 print "No commits applied."
2090 print "Applied only the commits marked with '*':"
2096 print star, read_pipe(["git", "show", "-s",
2097 "--format=format:%h %s", c])
2098 print "You will have to do 'git p4 sync' and rebase."
2100 if gitConfigBool("git-p4.exportLabels"):
2101 self.exportLabels = True
2103 if self.exportLabels:
2104 p4Labels = getP4Labels(self.depotPath)
2105 gitTags = getGitTags()
2107 missingGitTags = gitTags - p4Labels
2108 self.exportGitTags(missingGitTags)
2110 # exit with error unless everything applied perfectly
2111 if len(commits) != len(applied):
2117 """Represent a p4 view ("p4 help views"), and map files in a
2118 repo according to the view."""
2120 def __init__(self, client_name):
2122 self.client_prefix = "//%s/" % client_name
2123 # cache results of "p4 where" to lookup client file locations
2124 self.client_spec_path_cache = {}
2126 def append(self, view_line):
2127 """Parse a view line, splitting it into depot and client
2128 sides. Append to self.mappings, preserving order. This
2129 is only needed for tag creation."""
2131 # Split the view line into exactly two words. P4 enforces
2132 # structure on these lines that simplifies this quite a bit.
2134 # Either or both words may be double-quoted.
2135 # Single quotes do not matter.
2136 # Double-quote marks cannot occur inside the words.
2137 # A + or - prefix is also inside the quotes.
2138 # There are no quotes unless they contain a space.
2139 # The line is already white-space stripped.
2140 # The two words are separated by a single space.
2142 if view_line[0] == '"':
2143 # First word is double quoted. Find its end.
2144 close_quote_index = view_line.find('"', 1)
2145 if close_quote_index <= 0:
2146 die("No first-word closing quote found: %s" % view_line)
2147 depot_side = view_line[1:close_quote_index]
2148 # skip closing quote and space
2149 rhs_index = close_quote_index + 1 + 1
2151 space_index = view_line.find(" ")
2152 if space_index <= 0:
2153 die("No word-splitting space found: %s" % view_line)
2154 depot_side = view_line[0:space_index]
2155 rhs_index = space_index + 1
2157 # prefix + means overlay on previous mapping
2158 if depot_side.startswith("+"):
2159 depot_side = depot_side[1:]
2161 # prefix - means exclude this path, leave out of mappings
2163 if depot_side.startswith("-"):
2165 depot_side = depot_side[1:]
2168 self.mappings.append(depot_side)
2170 def convert_client_path(self, clientFile):
2171 # chop off //client/ part to make it relative
2172 if not clientFile.startswith(self.client_prefix):
2173 die("No prefix '%s' on clientFile '%s'" %
2174 (self.client_prefix, clientFile))
2175 return clientFile[len(self.client_prefix):]
2177 def update_client_spec_path_cache(self, files):
2178 """ Caching file paths by "p4 where" batch query """
2180 # List depot file paths exclude that already cached
2181 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2183 if len(fileArgs) == 0:
2184 return # All files in cache
2186 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2187 for res in where_result:
2188 if "code" in res and res["code"] == "error":
2189 # assume error is "... file(s) not in client view"
2191 if "clientFile" not in res:
2192 die("No clientFile in 'p4 where' output")
2194 # it will list all of them, but only one not unmap-ped
2196 if gitConfigBool("core.ignorecase"):
2197 res['depotFile'] = res['depotFile'].lower()
2198 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2200 # not found files or unmap files set to ""
2201 for depotFile in fileArgs:
2202 if gitConfigBool("core.ignorecase"):
2203 depotFile = depotFile.lower()
2204 if depotFile not in self.client_spec_path_cache:
2205 self.client_spec_path_cache[depotFile] = ""
2207 def map_in_client(self, depot_path):
2208 """Return the relative location in the client where this
2209 depot file should live. Returns "" if the file should
2210 not be mapped in the client."""
2212 if gitConfigBool("core.ignorecase"):
2213 depot_path = depot_path.lower()
2215 if depot_path in self.client_spec_path_cache:
2216 return self.client_spec_path_cache[depot_path]
2218 die( "Error: %s is not found in client spec path" % depot_path )
2221 class P4Sync(Command, P4UserMap):
2222 delete_actions = ( "delete", "move/delete", "purge" )
2225 Command.__init__(self)
2226 P4UserMap.__init__(self)
2228 optparse.make_option("--branch", dest="branch"),
2229 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2230 optparse.make_option("--changesfile", dest="changesFile"),
2231 optparse.make_option("--silent", dest="silent", action="store_true"),
2232 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2233 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2234 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2235 help="Import into refs/heads/ , not refs/remotes"),
2236 optparse.make_option("--max-changes", dest="maxChanges",
2237 help="Maximum number of changes to import"),
2238 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2239 help="Internal block size to use when iteratively calling p4 changes"),
2240 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2241 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2242 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2243 help="Only sync files that are included in the Perforce Client Spec"),
2244 optparse.make_option("-/", dest="cloneExclude",
2245 action="append", type="string",
2246 help="exclude depot path"),
2248 self.description = """Imports from Perforce into a git repository.\n
2250 //depot/my/project/ -- to import the current head
2251 //depot/my/project/@all -- to import everything
2252 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2254 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2256 self.usage += " //depot/path[@revRange]"
2258 self.createdBranches = set()
2259 self.committedChanges = set()
2261 self.detectBranches = False
2262 self.detectLabels = False
2263 self.importLabels = False
2264 self.changesFile = ""
2265 self.syncWithOrigin = True
2266 self.importIntoRemotes = True
2267 self.maxChanges = ""
2268 self.changes_block_size = None
2269 self.keepRepoPath = False
2270 self.depotPaths = None
2271 self.p4BranchesInGit = []
2272 self.cloneExclude = []
2273 self.useClientSpec = False
2274 self.useClientSpec_from_options = False
2275 self.clientSpecDirs = None
2276 self.tempBranches = []
2277 self.tempBranchLocation = "refs/git-p4-tmp"
2278 self.largeFileSystem = None
2280 if gitConfig('git-p4.largeFileSystem'):
2281 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2282 self.largeFileSystem = largeFileSystemConstructor(
2283 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2286 if gitConfig("git-p4.syncFromOrigin") == "false":
2287 self.syncWithOrigin = False
2289 # This is required for the "append" cloneExclude action
2290 def ensure_value(self, attr, value):
2291 if not hasattr(self, attr) or getattr(self, attr) is None:
2292 setattr(self, attr, value)
2293 return getattr(self, attr)
2295 # Force a checkpoint in fast-import and wait for it to finish
2296 def checkpoint(self):
2297 self.gitStream.write("checkpoint\n\n")
2298 self.gitStream.write("progress checkpoint\n\n")
2299 out = self.gitOutput.readline()
2301 print "checkpoint finished: " + out
2303 def extractFilesFromCommit(self, commit):
2304 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2305 for path in self.cloneExclude]
2308 while commit.has_key("depotFile%s" % fnum):
2309 path = commit["depotFile%s" % fnum]
2311 if [p for p in self.cloneExclude
2312 if p4PathStartsWith(path, p)]:
2315 found = [p for p in self.depotPaths
2316 if p4PathStartsWith(path, p)]
2323 file["rev"] = commit["rev%s" % fnum]
2324 file["action"] = commit["action%s" % fnum]
2325 file["type"] = commit["type%s" % fnum]
2330 def extractJobsFromCommit(self, commit):
2333 while commit.has_key("job%s" % jnum):
2334 job = commit["job%s" % jnum]
2339 def stripRepoPath(self, path, prefixes):
2340 """When streaming files, this is called to map a p4 depot path
2341 to where it should go in git. The prefixes are either
2342 self.depotPaths, or self.branchPrefixes in the case of
2343 branch detection."""
2345 if self.useClientSpec:
2346 # branch detection moves files up a level (the branch name)
2347 # from what client spec interpretation gives
2348 path = self.clientSpecDirs.map_in_client(path)
2349 if self.detectBranches:
2350 for b in self.knownBranches:
2351 if path.startswith(b + "/"):
2352 path = path[len(b)+1:]
2354 elif self.keepRepoPath:
2355 # Preserve everything in relative path name except leading
2356 # //depot/; just look at first prefix as they all should
2357 # be in the same depot.
2358 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2359 if p4PathStartsWith(path, depot):
2360 path = path[len(depot):]
2364 if p4PathStartsWith(path, p):
2365 path = path[len(p):]
2368 path = wildcard_decode(path)
2371 def splitFilesIntoBranches(self, commit):
2372 """Look at each depotFile in the commit to figure out to what
2373 branch it belongs."""
2375 if self.clientSpecDirs:
2376 files = self.extractFilesFromCommit(commit)
2377 self.clientSpecDirs.update_client_spec_path_cache(files)
2381 while commit.has_key("depotFile%s" % fnum):
2382 path = commit["depotFile%s" % fnum]
2383 found = [p for p in self.depotPaths
2384 if p4PathStartsWith(path, p)]
2391 file["rev"] = commit["rev%s" % fnum]
2392 file["action"] = commit["action%s" % fnum]
2393 file["type"] = commit["type%s" % fnum]
2396 # start with the full relative path where this file would
2398 if self.useClientSpec:
2399 relPath = self.clientSpecDirs.map_in_client(path)
2401 relPath = self.stripRepoPath(path, self.depotPaths)
2403 for branch in self.knownBranches.keys():
2404 # add a trailing slash so that a commit into qt/4.2foo
2405 # doesn't end up in qt/4.2, e.g.
2406 if relPath.startswith(branch + "/"):
2407 if branch not in branches:
2408 branches[branch] = []
2409 branches[branch].append(file)
2414 def writeToGitStream(self, gitMode, relPath, contents):
2415 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2416 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2418 self.gitStream.write(d)
2419 self.gitStream.write('\n')
2421 # output one file from the P4 stream
2422 # - helper for streamP4Files
2424 def streamOneP4File(self, file, contents):
2425 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2427 size = int(self.stream_file['fileSize'])
2428 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2431 (type_base, type_mods) = split_p4_type(file["type"])
2434 if "x" in type_mods:
2436 if type_base == "symlink":
2438 # p4 print on a symlink sometimes contains "target\n";
2439 # if it does, remove the newline
2440 data = ''.join(contents)
2442 # Some version of p4 allowed creating a symlink that pointed
2443 # to nothing. This causes p4 errors when checking out such
2444 # a change, and errors here too. Work around it by ignoring
2445 # the bad symlink; hopefully a future change fixes it.
2446 print "\nIgnoring empty symlink in %s" % file['depotFile']
2448 elif data[-1] == '\n':
2449 contents = [data[:-1]]
2453 if type_base == "utf16":
2454 # p4 delivers different text in the python output to -G
2455 # than it does when using "print -o", or normal p4 client
2456 # operations. utf16 is converted to ascii or utf8, perhaps.
2457 # But ascii text saved as -t utf16 is completely mangled.
2458 # Invoke print -o to get the real contents.
2460 # On windows, the newlines will always be mangled by print, so put
2461 # them back too. This is not needed to the cygwin windows version,
2462 # just the native "NT" type.
2465 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2466 except Exception as e:
2467 if 'Translation of file content failed' in str(e):
2468 type_base = 'binary'
2472 if p4_version_string().find('/NT') >= 0:
2473 text = text.replace('\r\n', '\n')
2476 if type_base == "apple":
2477 # Apple filetype files will be streamed as a concatenation of
2478 # its appledouble header and the contents. This is useless
2479 # on both macs and non-macs. If using "print -q -o xx", it
2480 # will create "xx" with the data, and "%xx" with the header.
2481 # This is also not very useful.
2483 # Ideally, someday, this script can learn how to generate
2484 # appledouble files directly and import those to git, but
2485 # non-mac machines can never find a use for apple filetype.
2486 print "\nIgnoring apple filetype file %s" % file['depotFile']
2489 # Note that we do not try to de-mangle keywords on utf16 files,
2490 # even though in theory somebody may want that.
2491 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2493 regexp = re.compile(pattern, re.VERBOSE)
2494 text = ''.join(contents)
2495 text = regexp.sub(r'$\1$', text)
2499 relPath.decode('ascii')
2502 if gitConfig('git-p4.pathEncoding'):
2503 encoding = gitConfig('git-p4.pathEncoding')
2504 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2506 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2508 if self.largeFileSystem:
2509 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2511 self.writeToGitStream(git_mode, relPath, contents)
2513 def streamOneP4Deletion(self, file):
2514 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2516 sys.stdout.write("delete %s\n" % relPath)
2518 self.gitStream.write("D %s\n" % relPath)
2520 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2521 self.largeFileSystem.removeLargeFile(relPath)
2523 # handle another chunk of streaming data
2524 def streamP4FilesCb(self, marshalled):
2526 # catch p4 errors and complain
2528 if "code" in marshalled:
2529 if marshalled["code"] == "error":
2530 if "data" in marshalled:
2531 err = marshalled["data"].rstrip()
2533 if not err and 'fileSize' in self.stream_file:
2534 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2535 if required_bytes > 0:
2536 err = 'Not enough space left on %s! Free at least %i MB.' % (
2537 os.getcwd(), required_bytes/1024/1024
2542 if self.stream_have_file_info:
2543 if "depotFile" in self.stream_file:
2544 f = self.stream_file["depotFile"]
2545 # force a failure in fast-import, else an empty
2546 # commit will be made
2547 self.gitStream.write("\n")
2548 self.gitStream.write("die-now\n")
2549 self.gitStream.close()
2550 # ignore errors, but make sure it exits first
2551 self.importProcess.wait()
2553 die("Error from p4 print for %s: %s" % (f, err))
2555 die("Error from p4 print: %s" % err)
2557 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2558 # start of a new file - output the old one first
2559 self.streamOneP4File(self.stream_file, self.stream_contents)
2560 self.stream_file = {}
2561 self.stream_contents = []
2562 self.stream_have_file_info = False
2564 # pick up the new file information... for the
2565 # 'data' field we need to append to our array
2566 for k in marshalled.keys():
2568 if 'streamContentSize' not in self.stream_file:
2569 self.stream_file['streamContentSize'] = 0
2570 self.stream_file['streamContentSize'] += len(marshalled['data'])
2571 self.stream_contents.append(marshalled['data'])
2573 self.stream_file[k] = marshalled[k]
2576 'streamContentSize' in self.stream_file and
2577 'fileSize' in self.stream_file and
2578 'depotFile' in self.stream_file):
2579 size = int(self.stream_file["fileSize"])
2581 progress = 100*self.stream_file['streamContentSize']/size
2582 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2585 self.stream_have_file_info = True
2587 # Stream directly from "p4 files" into "git fast-import"
2588 def streamP4Files(self, files):
2594 filesForCommit.append(f)
2595 if f['action'] in self.delete_actions:
2596 filesToDelete.append(f)
2598 filesToRead.append(f)
2601 for f in filesToDelete:
2602 self.streamOneP4Deletion(f)
2604 if len(filesToRead) > 0:
2605 self.stream_file = {}
2606 self.stream_contents = []
2607 self.stream_have_file_info = False
2609 # curry self argument
2610 def streamP4FilesCbSelf(entry):
2611 self.streamP4FilesCb(entry)
2613 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2615 p4CmdList(["-x", "-", "print"],
2617 cb=streamP4FilesCbSelf)
2620 if self.stream_file.has_key('depotFile'):
2621 self.streamOneP4File(self.stream_file, self.stream_contents)
2623 def make_email(self, userid):
2624 if userid in self.users:
2625 return self.users[userid]
2627 return "%s <a@b>" % userid
2629 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2630 """ Stream a p4 tag.
2631 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2635 print "writing tag %s for commit %s" % (labelName, commit)
2636 gitStream.write("tag %s\n" % labelName)
2637 gitStream.write("from %s\n" % commit)
2639 if labelDetails.has_key('Owner'):
2640 owner = labelDetails["Owner"]
2644 # Try to use the owner of the p4 label, or failing that,
2645 # the current p4 user id.
2647 email = self.make_email(owner)
2649 email = self.make_email(self.p4UserId())
2650 tagger = "%s %s %s" % (email, epoch, self.tz)
2652 gitStream.write("tagger %s\n" % tagger)
2654 print "labelDetails=",labelDetails
2655 if labelDetails.has_key('Description'):
2656 description = labelDetails['Description']
2658 description = 'Label from git p4'
2660 gitStream.write("data %d\n" % len(description))
2661 gitStream.write(description)
2662 gitStream.write("\n")
2664 def inClientSpec(self, path):
2665 if not self.clientSpecDirs:
2667 inClientSpec = self.clientSpecDirs.map_in_client(path)
2668 if not inClientSpec and self.verbose:
2669 print('Ignoring file outside of client spec: {0}'.format(path))
2672 def hasBranchPrefix(self, path):
2673 if not self.branchPrefixes:
2675 hasPrefix = [p for p in self.branchPrefixes
2676 if p4PathStartsWith(path, p)]
2677 if not hasPrefix and self.verbose:
2678 print('Ignoring file outside of prefix: {0}'.format(path))
2681 def commit(self, details, files, branch, parent = ""):
2682 epoch = details["time"]
2683 author = details["user"]
2684 jobs = self.extractJobsFromCommit(details)
2687 print('commit into {0}'.format(branch))
2689 if self.clientSpecDirs:
2690 self.clientSpecDirs.update_client_spec_path_cache(files)
2692 files = [f for f in files
2693 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2695 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2696 print('Ignoring revision {0} as it would produce an empty commit.'
2697 .format(details['change']))
2700 self.gitStream.write("commit %s\n" % branch)
2701 self.gitStream.write("mark :%s\n" % details["change"])
2702 self.committedChanges.add(int(details["change"]))
2704 if author not in self.users:
2705 self.getUserMapFromPerforceServer()
2706 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2708 self.gitStream.write("committer %s\n" % committer)
2710 self.gitStream.write("data <<EOT\n")
2711 self.gitStream.write(details["desc"])
2713 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2714 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2715 (','.join(self.branchPrefixes), details["change"]))
2716 if len(details['options']) > 0:
2717 self.gitStream.write(": options = %s" % details['options'])
2718 self.gitStream.write("]\nEOT\n\n")
2722 print "parent %s" % parent
2723 self.gitStream.write("from %s\n" % parent)
2725 self.streamP4Files(files)
2726 self.gitStream.write("\n")
2728 change = int(details["change"])
2730 if self.labels.has_key(change):
2731 label = self.labels[change]
2732 labelDetails = label[0]
2733 labelRevisions = label[1]
2735 print "Change %s is labelled %s" % (change, labelDetails)
2737 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2738 for p in self.branchPrefixes])
2740 if len(files) == len(labelRevisions):
2744 if info["action"] in self.delete_actions:
2746 cleanedFiles[info["depotFile"]] = info["rev"]
2748 if cleanedFiles == labelRevisions:
2749 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2753 print ("Tag %s does not match with change %s: files do not match."
2754 % (labelDetails["label"], change))
2758 print ("Tag %s does not match with change %s: file count is different."
2759 % (labelDetails["label"], change))
2761 # Build a dictionary of changelists and labels, for "detect-labels" option.
2762 def getLabels(self):
2765 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2766 if len(l) > 0 and not self.silent:
2767 print "Finding files belonging to labels in %s" % `self.depotPaths`
2770 label = output["label"]
2774 print "Querying files for label %s" % label
2775 for file in p4CmdList(["files"] +
2776 ["%s...@%s" % (p, label)
2777 for p in self.depotPaths]):
2778 revisions[file["depotFile"]] = file["rev"]
2779 change = int(file["change"])
2780 if change > newestChange:
2781 newestChange = change
2783 self.labels[newestChange] = [output, revisions]
2786 print "Label changes: %s" % self.labels.keys()
2788 # Import p4 labels as git tags. A direct mapping does not
2789 # exist, so assume that if all the files are at the same revision
2790 # then we can use that, or it's something more complicated we should
2792 def importP4Labels(self, stream, p4Labels):
2794 print "import p4 labels: " + ' '.join(p4Labels)
2796 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2797 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2798 if len(validLabelRegexp) == 0:
2799 validLabelRegexp = defaultLabelRegexp
2800 m = re.compile(validLabelRegexp)
2802 for name in p4Labels:
2805 if not m.match(name):
2807 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2810 if name in ignoredP4Labels:
2813 labelDetails = p4CmdList(['label', "-o", name])[0]
2815 # get the most recent changelist for each file in this label
2816 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2817 for p in self.depotPaths])
2819 if change.has_key('change'):
2820 # find the corresponding git commit; take the oldest commit
2821 changelist = int(change['change'])
2822 if changelist in self.committedChanges:
2823 gitCommit = ":%d" % changelist # use a fast-import mark
2826 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2827 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2828 if len(gitCommit) == 0:
2829 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2832 gitCommit = gitCommit.strip()
2835 # Convert from p4 time format
2837 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2839 print "Could not convert label time %s" % labelDetails['Update']
2842 when = int(time.mktime(tmwhen))
2843 self.streamTag(stream, name, labelDetails, gitCommit, when)
2845 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2848 print "Label %s has no changelists - possibly deleted?" % name
2851 # We can't import this label; don't try again as it will get very
2852 # expensive repeatedly fetching all the files for labels that will
2853 # never be imported. If the label is moved in the future, the
2854 # ignore will need to be removed manually.
2855 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2857 def guessProjectName(self):
2858 for p in self.depotPaths:
2861 p = p[p.strip().rfind("/") + 1:]
2862 if not p.endswith("/"):
2866 def getBranchMapping(self):
2867 lostAndFoundBranches = set()
2869 user = gitConfig("git-p4.branchUser")
2871 command = "branches -u %s" % user
2873 command = "branches"
2875 for info in p4CmdList(command):
2876 details = p4Cmd(["branch", "-o", info["branch"]])
2878 while details.has_key("View%s" % viewIdx):
2879 paths = details["View%s" % viewIdx].split(" ")
2880 viewIdx = viewIdx + 1
2881 # require standard //depot/foo/... //depot/bar/... mapping
2882 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2885 destination = paths[1]
2887 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2888 source = source[len(self.depotPaths[0]):-4]
2889 destination = destination[len(self.depotPaths[0]):-4]
2891 if destination in self.knownBranches:
2893 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2894 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2897 self.knownBranches[destination] = source
2899 lostAndFoundBranches.discard(destination)
2901 if source not in self.knownBranches:
2902 lostAndFoundBranches.add(source)
2904 # Perforce does not strictly require branches to be defined, so we also
2905 # check git config for a branch list.
2907 # Example of branch definition in git config file:
2909 # branchList=main:branchA
2910 # branchList=main:branchB
2911 # branchList=branchA:branchC
2912 configBranches = gitConfigList("git-p4.branchList")
2913 for branch in configBranches:
2915 (source, destination) = branch.split(":")
2916 self.knownBranches[destination] = source
2918 lostAndFoundBranches.discard(destination)
2920 if source not in self.knownBranches:
2921 lostAndFoundBranches.add(source)
2924 for branch in lostAndFoundBranches:
2925 self.knownBranches[branch] = branch
2927 def getBranchMappingFromGitBranches(self):
2928 branches = p4BranchesInGit(self.importIntoRemotes)
2929 for branch in branches.keys():
2930 if branch == "master":
2933 branch = branch[len(self.projectName):]
2934 self.knownBranches[branch] = branch
2936 def updateOptionDict(self, d):
2938 if self.keepRepoPath:
2939 option_keys['keepRepoPath'] = 1
2941 d["options"] = ' '.join(sorted(option_keys.keys()))
2943 def readOptions(self, d):
2944 self.keepRepoPath = (d.has_key('options')
2945 and ('keepRepoPath' in d['options']))
2947 def gitRefForBranch(self, branch):
2948 if branch == "main":
2949 return self.refPrefix + "master"
2951 if len(branch) <= 0:
2954 return self.refPrefix + self.projectName + branch
2956 def gitCommitByP4Change(self, ref, change):
2958 print "looking in ref " + ref + " for change %s using bisect..." % change
2961 latestCommit = parseRevision(ref)
2965 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2966 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2971 log = extractLogMessageFromGitCommit(next)
2972 settings = extractSettingsGitLog(log)
2973 currentChange = int(settings['change'])
2975 print "current change %s" % currentChange
2977 if currentChange == change:
2979 print "found %s" % next
2982 if currentChange < change:
2983 earliestCommit = "^%s" % next
2985 latestCommit = "%s" % next
2989 def importNewBranch(self, branch, maxChange):
2990 # make fast-import flush all changes to disk and update the refs using the checkpoint
2991 # command so that we can try to find the branch parent in the git history
2992 self.gitStream.write("checkpoint\n\n");
2993 self.gitStream.flush();
2994 branchPrefix = self.depotPaths[0] + branch + "/"
2995 range = "@1,%s" % maxChange
2996 #print "prefix" + branchPrefix
2997 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2998 if len(changes) <= 0:
3000 firstChange = changes[0]
3001 #print "first change in branch: %s" % firstChange
3002 sourceBranch = self.knownBranches[branch]
3003 sourceDepotPath = self.depotPaths[0] + sourceBranch
3004 sourceRef = self.gitRefForBranch(sourceBranch)
3005 #print "source " + sourceBranch
3007 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3008 #print "branch parent: %s" % branchParentChange
3009 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3010 if len(gitParent) > 0:
3011 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3012 #print "parent git commit: %s" % gitParent
3014 self.importChanges(changes)
3017 def searchParent(self, parent, branch, target):
3019 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3020 "--no-merges", parent]):
3022 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3025 print "Found parent of %s in commit %s" % (branch, blob)
3032 def importChanges(self, changes):
3034 for change in changes:
3035 description = p4_describe(change)
3036 self.updateOptionDict(description)
3039 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3044 if self.detectBranches:
3045 branches = self.splitFilesIntoBranches(description)
3046 for branch in branches.keys():
3048 branchPrefix = self.depotPaths[0] + branch + "/"
3049 self.branchPrefixes = [ branchPrefix ]
3053 filesForCommit = branches[branch]
3056 print "branch is %s" % branch
3058 self.updatedBranches.add(branch)
3060 if branch not in self.createdBranches:
3061 self.createdBranches.add(branch)
3062 parent = self.knownBranches[branch]
3063 if parent == branch:
3066 fullBranch = self.projectName + branch
3067 if fullBranch not in self.p4BranchesInGit:
3069 print("\n Importing new branch %s" % fullBranch);
3070 if self.importNewBranch(branch, change - 1):
3072 self.p4BranchesInGit.append(fullBranch)
3074 print("\n Resuming with change %s" % change);
3077 print "parent determined through known branches: %s" % parent
3079 branch = self.gitRefForBranch(branch)
3080 parent = self.gitRefForBranch(parent)
3083 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3085 if len(parent) == 0 and branch in self.initialParents:
3086 parent = self.initialParents[branch]
3087 del self.initialParents[branch]
3091 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3093 print "Creating temporary branch: " + tempBranch
3094 self.commit(description, filesForCommit, tempBranch)
3095 self.tempBranches.append(tempBranch)
3097 blob = self.searchParent(parent, branch, tempBranch)
3099 self.commit(description, filesForCommit, branch, blob)
3102 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3103 self.commit(description, filesForCommit, branch, parent)
3105 files = self.extractFilesFromCommit(description)
3106 self.commit(description, files, self.branch,
3108 # only needed once, to connect to the previous commit
3109 self.initialParent = ""
3111 print self.gitError.read()
3114 def importHeadRevision(self, revision):
3115 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3118 details["user"] = "git perforce import user"
3119 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3120 % (' '.join(self.depotPaths), revision))
3121 details["change"] = revision
3125 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3127 for info in p4CmdList(["files"] + fileArgs):
3129 if 'code' in info and info['code'] == 'error':
3130 sys.stderr.write("p4 returned an error: %s\n"
3132 if info['data'].find("must refer to client") >= 0:
3133 sys.stderr.write("This particular p4 error is misleading.\n")
3134 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3135 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3137 if 'p4ExitCode' in info:
3138 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3142 change = int(info["change"])
3143 if change > newestRevision:
3144 newestRevision = change
3146 if info["action"] in self.delete_actions:
3147 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3148 #fileCnt = fileCnt + 1
3151 for prop in ["depotFile", "rev", "action", "type" ]:
3152 details["%s%s" % (prop, fileCnt)] = info[prop]
3154 fileCnt = fileCnt + 1
3156 details["change"] = newestRevision
3158 # Use time from top-most change so that all git p4 clones of
3159 # the same p4 repo have the same commit SHA1s.
3160 res = p4_describe(newestRevision)
3161 details["time"] = res["time"]
3163 self.updateOptionDict(details)
3165 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3167 print "IO error with git fast-import. Is your git version recent enough?"
3168 print self.gitError.read()
3171 def run(self, args):
3172 self.depotPaths = []
3173 self.changeRange = ""
3174 self.previousDepotPaths = []
3175 self.hasOrigin = False
3177 # map from branch depot path to parent branch
3178 self.knownBranches = {}
3179 self.initialParents = {}
3181 if self.importIntoRemotes:
3182 self.refPrefix = "refs/remotes/p4/"
3184 self.refPrefix = "refs/heads/p4/"
3186 if self.syncWithOrigin:
3187 self.hasOrigin = originP4BranchesExist()
3190 print 'Syncing with origin first, using "git fetch origin"'
3191 system("git fetch origin")
3193 branch_arg_given = bool(self.branch)
3194 if len(self.branch) == 0:
3195 self.branch = self.refPrefix + "master"
3196 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3197 system("git update-ref %s refs/heads/p4" % self.branch)
3198 system("git branch -D p4")
3200 # accept either the command-line option, or the configuration variable
3201 if self.useClientSpec:
3202 # will use this after clone to set the variable
3203 self.useClientSpec_from_options = True
3205 if gitConfigBool("git-p4.useclientspec"):
3206 self.useClientSpec = True
3207 if self.useClientSpec:
3208 self.clientSpecDirs = getClientSpec()
3210 # TODO: should always look at previous commits,
3211 # merge with previous imports, if possible.
3214 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3216 # branches holds mapping from branch name to sha1
3217 branches = p4BranchesInGit(self.importIntoRemotes)
3219 # restrict to just this one, disabling detect-branches
3220 if branch_arg_given:
3221 short = self.branch.split("/")[-1]
3222 if short in branches:
3223 self.p4BranchesInGit = [ short ]
3225 self.p4BranchesInGit = branches.keys()
3227 if len(self.p4BranchesInGit) > 1:
3229 print "Importing from/into multiple branches"
3230 self.detectBranches = True
3231 for branch in branches.keys():
3232 self.initialParents[self.refPrefix + branch] = \
3236 print "branches: %s" % self.p4BranchesInGit
3239 for branch in self.p4BranchesInGit:
3240 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3242 settings = extractSettingsGitLog(logMsg)
3244 self.readOptions(settings)
3245 if (settings.has_key('depot-paths')
3246 and settings.has_key ('change')):
3247 change = int(settings['change']) + 1
3248 p4Change = max(p4Change, change)
3250 depotPaths = sorted(settings['depot-paths'])
3251 if self.previousDepotPaths == []:
3252 self.previousDepotPaths = depotPaths
3255 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3256 prev_list = prev.split("/")
3257 cur_list = cur.split("/")
3258 for i in range(0, min(len(cur_list), len(prev_list))):
3259 if cur_list[i] <> prev_list[i]:
3263 paths.append ("/".join(cur_list[:i + 1]))
3265 self.previousDepotPaths = paths
3268 self.depotPaths = sorted(self.previousDepotPaths)
3269 self.changeRange = "@%s,#head" % p4Change
3270 if not self.silent and not self.detectBranches:
3271 print "Performing incremental import into %s git branch" % self.branch
3273 # accept multiple ref name abbreviations:
3274 # refs/foo/bar/branch -> use it exactly
3275 # p4/branch -> prepend refs/remotes/ or refs/heads/
3276 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3277 if not self.branch.startswith("refs/"):
3278 if self.importIntoRemotes:
3279 prepend = "refs/remotes/"
3281 prepend = "refs/heads/"
3282 if not self.branch.startswith("p4/"):
3284 self.branch = prepend + self.branch
3286 if len(args) == 0 and self.depotPaths:
3288 print "Depot paths: %s" % ' '.join(self.depotPaths)
3290 if self.depotPaths and self.depotPaths != args:
3291 print ("previous import used depot path %s and now %s was specified. "
3292 "This doesn't work!" % (' '.join (self.depotPaths),
3296 self.depotPaths = sorted(args)
3301 # Make sure no revision specifiers are used when --changesfile
3303 bad_changesfile = False
3304 if len(self.changesFile) > 0:
3305 for p in self.depotPaths:
3306 if p.find("@") >= 0 or p.find("#") >= 0:
3307 bad_changesfile = True
3310 die("Option --changesfile is incompatible with revision specifiers")
3313 for p in self.depotPaths:
3314 if p.find("@") != -1:
3315 atIdx = p.index("@")
3316 self.changeRange = p[atIdx:]
3317 if self.changeRange == "@all":
3318 self.changeRange = ""
3319 elif ',' not in self.changeRange:
3320 revision = self.changeRange
3321 self.changeRange = ""
3323 elif p.find("#") != -1:
3324 hashIdx = p.index("#")
3325 revision = p[hashIdx:]
3327 elif self.previousDepotPaths == []:
3328 # pay attention to changesfile, if given, else import
3329 # the entire p4 tree at the head revision
3330 if len(self.changesFile) == 0:
3333 p = re.sub ("\.\.\.$", "", p)
3334 if not p.endswith("/"):
3339 self.depotPaths = newPaths
3341 # --detect-branches may change this for each branch
3342 self.branchPrefixes = self.depotPaths
3344 self.loadUserMapFromCache()
3346 if self.detectLabels:
3349 if self.detectBranches:
3350 ## FIXME - what's a P4 projectName ?
3351 self.projectName = self.guessProjectName()
3354 self.getBranchMappingFromGitBranches()
3356 self.getBranchMapping()
3358 print "p4-git branches: %s" % self.p4BranchesInGit
3359 print "initial parents: %s" % self.initialParents
3360 for b in self.p4BranchesInGit:
3364 b = b[len(self.projectName):]
3365 self.createdBranches.add(b)
3367 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3369 self.importProcess = subprocess.Popen(["git", "fast-import"],
3370 stdin=subprocess.PIPE,
3371 stdout=subprocess.PIPE,
3372 stderr=subprocess.PIPE);
3373 self.gitOutput = self.importProcess.stdout
3374 self.gitStream = self.importProcess.stdin
3375 self.gitError = self.importProcess.stderr
3378 self.importHeadRevision(revision)
3382 if len(self.changesFile) > 0:
3383 output = open(self.changesFile).readlines()
3386 changeSet.add(int(line))
3388 for change in changeSet:
3389 changes.append(change)
3393 # catch "git p4 sync" with no new branches, in a repo that
3394 # does not have any existing p4 branches
3396 if not self.p4BranchesInGit:
3397 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3399 # The default branch is master, unless --branch is used to
3400 # specify something else. Make sure it exists, or complain
3401 # nicely about how to use --branch.
3402 if not self.detectBranches:
3403 if not branch_exists(self.branch):
3404 if branch_arg_given:
3405 die("Error: branch %s does not exist." % self.branch)
3407 die("Error: no branch %s; perhaps specify one with --branch." %
3411 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3413 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3415 if len(self.maxChanges) > 0:
3416 changes = changes[:min(int(self.maxChanges), len(changes))]
3418 if len(changes) == 0:
3420 print "No changes to import!"
3422 if not self.silent and not self.detectBranches:
3423 print "Import destination: %s" % self.branch
3425 self.updatedBranches = set()
3427 if not self.detectBranches:
3429 # start a new branch
3430 self.initialParent = ""
3432 # build on a previous revision
3433 self.initialParent = parseRevision(self.branch)
3435 self.importChanges(changes)
3439 if len(self.updatedBranches) > 0:
3440 sys.stdout.write("Updated branches: ")
3441 for b in self.updatedBranches:
3442 sys.stdout.write("%s " % b)
3443 sys.stdout.write("\n")
3445 if gitConfigBool("git-p4.importLabels"):
3446 self.importLabels = True
3448 if self.importLabels:
3449 p4Labels = getP4Labels(self.depotPaths)
3450 gitTags = getGitTags()
3452 missingP4Labels = p4Labels - gitTags
3453 self.importP4Labels(self.gitStream, missingP4Labels)
3455 self.gitStream.close()
3456 if self.importProcess.wait() != 0:
3457 die("fast-import failed: %s" % self.gitError.read())
3458 self.gitOutput.close()
3459 self.gitError.close()
3461 # Cleanup temporary branches created during import
3462 if self.tempBranches != []:
3463 for branch in self.tempBranches:
3464 read_pipe("git update-ref -d %s" % branch)
3465 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3467 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3468 # a convenient shortcut refname "p4".
3469 if self.importIntoRemotes:
3470 head_ref = self.refPrefix + "HEAD"
3471 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3472 system(["git", "symbolic-ref", head_ref, self.branch])
3476 class P4Rebase(Command):
3478 Command.__init__(self)
3480 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3482 self.importLabels = False
3483 self.description = ("Fetches the latest revision from perforce and "
3484 + "rebases the current work (branch) against it")
3486 def run(self, args):
3488 sync.importLabels = self.importLabels
3491 return self.rebase()
3494 if os.system("git update-index --refresh") != 0:
3495 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3496 if len(read_pipe("git diff-index HEAD --")) > 0:
3497 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3499 [upstream, settings] = findUpstreamBranchPoint()
3500 if len(upstream) == 0:
3501 die("Cannot find upstream branchpoint for rebase")
3503 # the branchpoint may be p4/foo~3, so strip off the parent
3504 upstream = re.sub("~[0-9]+$", "", upstream)
3506 print "Rebasing the current branch onto %s" % upstream
3507 oldHead = read_pipe("git rev-parse HEAD").strip()
3508 system("git rebase %s" % upstream)
3509 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3512 class P4Clone(P4Sync):
3514 P4Sync.__init__(self)
3515 self.description = "Creates a new git repository and imports from Perforce into it"
3516 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3518 optparse.make_option("--destination", dest="cloneDestination",
3519 action='store', default=None,
3520 help="where to leave result of the clone"),
3521 optparse.make_option("--bare", dest="cloneBare",
3522 action="store_true", default=False),
3524 self.cloneDestination = None
3525 self.needsGit = False
3526 self.cloneBare = False
3528 def defaultDestination(self, args):
3529 ## TODO: use common prefix of args?
3531 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3532 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3533 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3534 depotDir = re.sub(r"/$", "", depotDir)
3535 return os.path.split(depotDir)[1]
3537 def run(self, args):
3541 if self.keepRepoPath and not self.cloneDestination:
3542 sys.stderr.write("Must specify destination for --keep-path\n")
3547 if not self.cloneDestination and len(depotPaths) > 1:
3548 self.cloneDestination = depotPaths[-1]
3549 depotPaths = depotPaths[:-1]
3551 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3552 for p in depotPaths:
3553 if not p.startswith("//"):
3554 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3557 if not self.cloneDestination:
3558 self.cloneDestination = self.defaultDestination(args)
3560 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3562 if not os.path.exists(self.cloneDestination):
3563 os.makedirs(self.cloneDestination)
3564 chdir(self.cloneDestination)
3566 init_cmd = [ "git", "init" ]
3568 init_cmd.append("--bare")
3569 retcode = subprocess.call(init_cmd)
3571 raise CalledProcessError(retcode, init_cmd)
3573 if not P4Sync.run(self, depotPaths):
3576 # create a master branch and check out a work tree
3577 if gitBranchExists(self.branch):
3578 system([ "git", "branch", "master", self.branch ])
3579 if not self.cloneBare:
3580 system([ "git", "checkout", "-f" ])
3582 print 'Not checking out any branch, use ' \
3583 '"git checkout -q -b master <branch>"'
3585 # auto-set this variable if invoked with --use-client-spec
3586 if self.useClientSpec_from_options:
3587 system("git config --bool git-p4.useclientspec true")
3591 class P4Branches(Command):
3593 Command.__init__(self)
3595 self.description = ("Shows the git branches that hold imports and their "
3596 + "corresponding perforce depot paths")
3597 self.verbose = False
3599 def run(self, args):
3600 if originP4BranchesExist():
3601 createOrUpdateBranchesFromOrigin()
3603 cmdline = "git rev-parse --symbolic "
3604 cmdline += " --remotes"
3606 for line in read_pipe_lines(cmdline):
3609 if not line.startswith('p4/') or line == "p4/HEAD":
3613 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3614 settings = extractSettingsGitLog(log)
3616 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3619 class HelpFormatter(optparse.IndentedHelpFormatter):
3621 optparse.IndentedHelpFormatter.__init__(self)
3623 def format_description(self, description):
3625 return description + "\n"
3629 def printUsage(commands):
3630 print "usage: %s <command> [options]" % sys.argv[0]
3632 print "valid commands: %s" % ", ".join(commands)
3634 print "Try %s <command> --help for command specific help." % sys.argv[0]
3639 "submit" : P4Submit,
3640 "commit" : P4Submit,
3642 "rebase" : P4Rebase,
3644 "rollback" : P4RollBack,
3645 "branches" : P4Branches
3650 if len(sys.argv[1:]) == 0:
3651 printUsage(commands.keys())
3654 cmdName = sys.argv[1]
3656 klass = commands[cmdName]
3659 print "unknown command %s" % cmdName
3661 printUsage(commands.keys())
3664 options = cmd.options
3665 cmd.gitdir = os.environ.get("GIT_DIR", None)
3669 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3671 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3673 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3675 description = cmd.description,
3676 formatter = HelpFormatter())
3678 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3680 verbose = cmd.verbose
3682 if cmd.gitdir == None:
3683 cmd.gitdir = os.path.abspath(".git")
3684 if not isValidGitDir(cmd.gitdir):
3685 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3686 if os.path.exists(cmd.gitdir):
3687 cdup = read_pipe("git rev-parse --show-cdup").strip()
3691 if not isValidGitDir(cmd.gitdir):
3692 if isValidGitDir(cmd.gitdir + "/.git"):
3693 cmd.gitdir += "/.git"
3695 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3697 os.environ["GIT_DIR"] = cmd.gitdir
3699 if not cmd.run(args):
3704 if __name__ == '__main__':