3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
29 from subprocess import CalledProcessError
31 # from python2.7:subprocess.py
32 # Exception classes used by this module.
33 class CalledProcessError(Exception):
34 """This exception is raised when a process run by check_call() returns
35 a non-zero exit status. The exit status will be stored in the
36 returncode attribute."""
37 def __init__(self, returncode, cmd):
38 self.returncode = returncode
41 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
45 # Only labels/tags matching this will be imported/exported
46 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
48 # Grab changes in blocks of this many revisions, unless otherwise requested
49 defaultBlockSize = 512
51 def p4_build_cmd(cmd):
52 """Build a suitable p4 command line.
54 This consolidates building and returning a p4 command line into one
55 location. It means that hooking into the environment, or other configuration
56 can be done more easily.
60 user = gitConfig("git-p4.user")
62 real_cmd += ["-u",user]
64 password = gitConfig("git-p4.password")
66 real_cmd += ["-P", password]
68 port = gitConfig("git-p4.port")
70 real_cmd += ["-p", port]
72 host = gitConfig("git-p4.host")
74 real_cmd += ["-H", host]
76 client = gitConfig("git-p4.client")
78 real_cmd += ["-c", client]
81 if isinstance(cmd,basestring):
82 real_cmd = ' '.join(real_cmd) + ' ' + cmd
87 def chdir(path, is_client_path=False):
88 """Do chdir to the given path, and set the PWD environment
89 variable for use by P4. It does not look at getcwd() output.
90 Since we're not using the shell, it is necessary to set the
91 PWD environment variable explicitly.
93 Normally, expand the path to force it to be absolute. This
94 addresses the use of relative path names inside P4 settings,
95 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
96 as given; it looks for .p4config using PWD.
98 If is_client_path, the path was handed to us directly by p4,
99 and may be a symbolic link. Do not call os.getcwd() in this
100 case, because it will cause p4 to think that PWD is not inside
105 if not is_client_path:
107 os.environ['PWD'] = path
110 """Return free space in bytes on the disk of the given dirname."""
111 if platform.system() == 'Windows':
112 free_bytes = ctypes.c_ulonglong(0)
113 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
114 return free_bytes.value
116 st = os.statvfs(os.getcwd())
117 return st.f_bavail * st.f_frsize
123 sys.stderr.write(msg + "\n")
126 def write_pipe(c, stdin):
128 sys.stderr.write('Writing pipe: %s\n' % str(c))
130 expand = isinstance(c,basestring)
131 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133 val = pipe.write(stdin)
136 die('Command failed: %s' % str(c))
140 def p4_write_pipe(c, stdin):
141 real_cmd = p4_build_cmd(c)
142 return write_pipe(real_cmd, stdin)
144 def read_pipe(c, ignore_error=False):
146 sys.stderr.write('Reading pipe: %s\n' % str(c))
148 expand = isinstance(c,basestring)
149 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
150 (out, err) = p.communicate()
151 if p.returncode != 0 and not ignore_error:
152 die('Command failed: %s\nError: %s' % (str(c), err))
155 def p4_read_pipe(c, ignore_error=False):
156 real_cmd = p4_build_cmd(c)
157 return read_pipe(real_cmd, ignore_error)
159 def read_pipe_lines(c):
161 sys.stderr.write('Reading pipe: %s\n' % str(c))
163 expand = isinstance(c, basestring)
164 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166 val = pipe.readlines()
167 if pipe.close() or p.wait():
168 die('Command failed: %s' % str(c))
172 def p4_read_pipe_lines(c):
173 """Specifically invoke p4 on the command supplied. """
174 real_cmd = p4_build_cmd(c)
175 return read_pipe_lines(real_cmd)
177 def p4_has_command(cmd):
178 """Ask p4 for help on this command. If it returns an error, the
179 command does not exist in this version of p4."""
180 real_cmd = p4_build_cmd(["help", cmd])
181 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
182 stderr=subprocess.PIPE)
184 return p.returncode == 0
186 def p4_has_move_command():
187 """See if the move command exists, that it supports -k, and that
188 it has not been administratively disabled. The arguments
189 must be correct, but the filenames do not have to exist. Use
190 ones with wildcards so even if they exist, it will fail."""
192 if not p4_has_command("move"):
194 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
195 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
196 (out, err) = p.communicate()
197 # return code will be 1 in either case
198 if err.find("Invalid option") >= 0:
200 if err.find("disabled") >= 0:
202 # assume it failed because @... was invalid changelist
206 expand = isinstance(cmd,basestring)
208 sys.stderr.write("executing %s\n" % str(cmd))
209 retcode = subprocess.call(cmd, shell=expand)
211 raise CalledProcessError(retcode, cmd)
214 """Specifically invoke p4 as the system command. """
215 real_cmd = p4_build_cmd(cmd)
216 expand = isinstance(real_cmd, basestring)
217 retcode = subprocess.call(real_cmd, shell=expand)
219 raise CalledProcessError(retcode, real_cmd)
221 _p4_version_string = None
222 def p4_version_string():
223 """Read the version string, showing just the last line, which
224 hopefully is the interesting version bit.
227 Perforce - The Fast Software Configuration Management System.
228 Copyright 1995-2011 Perforce Software. All rights reserved.
229 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
231 global _p4_version_string
232 if not _p4_version_string:
233 a = p4_read_pipe_lines(["-V"])
234 _p4_version_string = a[-1].rstrip()
235 return _p4_version_string
237 def p4_integrate(src, dest):
238 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
240 def p4_sync(f, *options):
241 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
244 # forcibly add file names with wildcards
245 if wildcard_present(f):
246 p4_system(["add", "-f", f])
248 p4_system(["add", f])
251 p4_system(["delete", wildcard_encode(f)])
254 p4_system(["edit", wildcard_encode(f)])
257 p4_system(["revert", wildcard_encode(f)])
259 def p4_reopen(type, f):
260 p4_system(["reopen", "-t", type, wildcard_encode(f)])
262 def p4_move(src, dest):
263 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
265 def p4_last_change():
266 results = p4CmdList(["changes", "-m", "1"])
267 return int(results[0]['change'])
269 def p4_describe(change):
270 """Make sure it returns a valid result by checking for
271 the presence of field "time". Return a dict of the
274 ds = p4CmdList(["describe", "-s", str(change)])
276 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280 if "p4ExitCode" in d:
281 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
284 if d["code"] == "error":
285 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
288 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
293 # Canonicalize the p4 type and return a tuple of the
294 # base type, plus any modifiers. See "p4 help filetypes"
295 # for a list and explanation.
297 def split_p4_type(p4type):
299 p4_filetypes_historical = {
300 "ctempobj": "binary+Sw",
306 "tempobj": "binary+FSw",
307 "ubinary": "binary+F",
308 "uresource": "resource+F",
309 "uxbinary": "binary+Fx",
310 "xbinary": "binary+x",
312 "xtempobj": "binary+Swx",
314 "xunicode": "unicode+x",
317 if p4type in p4_filetypes_historical:
318 p4type = p4_filetypes_historical[p4type]
320 s = p4type.split("+")
328 # return the raw p4 type of a file (text, text+ko, etc)
331 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
332 return results[0]['headType']
335 # Given a type base and modifier, return a regexp matching
336 # the keywords that can be expanded in the file
338 def p4_keywords_regexp_for_type(base, type_mods):
339 if base in ("text", "unicode", "binary"):
341 if "ko" in type_mods:
343 elif "k" in type_mods:
344 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348 \$ # Starts with a dollar, followed by...
349 (%s) # one of the keywords, followed by...
350 (:[^$\n]+)? # possibly an old expansion, followed by...
358 # Given a file, return a regexp matching the possible
359 # RCS keywords that will be expanded, or None for files
360 # with kw expansion turned off.
362 def p4_keywords_regexp_for_file(file):
363 if not os.path.exists(file):
366 (type_base, type_mods) = split_p4_type(p4_type(file))
367 return p4_keywords_regexp_for_type(type_base, type_mods)
369 def setP4ExecBit(file, mode):
370 # Reopens an already open file and changes the execute bit to match
371 # the execute bit setting in the passed in mode.
375 if not isModeExec(mode):
376 p4Type = getP4OpenedType(file)
377 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
378 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
379 if p4Type[-1] == "+":
380 p4Type = p4Type[0:-1]
382 p4_reopen(p4Type, file)
384 def getP4OpenedType(file):
385 # Returns the perforce file type for the given file.
387 result = p4_read_pipe(["opened", wildcard_encode(file)])
388 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
390 return match.group(1)
392 die("Could not determine file type for %s (result: '%s')" % (file, result))
394 # Return the set of all p4 labels
395 def getP4Labels(depotPaths):
397 if isinstance(depotPaths,basestring):
398 depotPaths = [depotPaths]
400 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
406 # Return the set of all git tags
409 for line in read_pipe_lines(["git", "tag"]):
414 def diffTreePattern():
415 # This is a simple generator for the diff tree regex pattern. This could be
416 # a class variable if this and parseDiffTreeEntry were a part of a class.
417 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421 def parseDiffTreeEntry(entry):
422 """Parses a single diff tree entry into its component elements.
424 See git-diff-tree(1) manpage for details about the format of the diff
425 output. This method returns a dictionary with the following elements:
427 src_mode - The mode of the source file
428 dst_mode - The mode of the destination file
429 src_sha1 - The sha1 for the source file
430 dst_sha1 - The sha1 fr the destination file
431 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
432 status_score - The score for the status (applicable for 'C' and 'R'
433 statuses). This is None if there is no score.
434 src - The path for the source file.
435 dst - The path for the destination file. This is only present for
436 copy or renames. If it is not present, this is None.
438 If the pattern is not matched, None is returned."""
440 match = diffTreePattern().next().match(entry)
443 'src_mode': match.group(1),
444 'dst_mode': match.group(2),
445 'src_sha1': match.group(3),
446 'dst_sha1': match.group(4),
447 'status': match.group(5),
448 'status_score': match.group(6),
449 'src': match.group(7),
450 'dst': match.group(10)
454 def isModeExec(mode):
455 # Returns True if the given git mode represents an executable file,
457 return mode[-3:] == "755"
459 def isModeExecChanged(src_mode, dst_mode):
460 return isModeExec(src_mode) != isModeExec(dst_mode)
462 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
464 if isinstance(cmd,basestring):
471 cmd = p4_build_cmd(cmd)
473 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
475 # Use a temporary file to avoid deadlocks without
476 # subprocess.communicate(), which would put another copy
477 # of stdout into memory.
479 if stdin is not None:
480 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
481 if isinstance(stdin,basestring):
482 stdin_file.write(stdin)
485 stdin_file.write(i + '\n')
489 p4 = subprocess.Popen(cmd,
492 stdout=subprocess.PIPE)
497 entry = marshal.load(p4.stdout)
507 entry["p4ExitCode"] = exitCode
513 list = p4CmdList(cmd)
519 def p4Where(depotPath):
520 if not depotPath.endswith("/"):
522 depotPathLong = depotPath + "..."
523 outputList = p4CmdList(["where", depotPathLong])
525 for entry in outputList:
526 if "depotFile" in entry:
527 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
528 # The base path always ends with "/...".
529 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
532 elif "data" in entry:
533 data = entry.get("data")
534 space = data.find(" ")
535 if data[:space] == depotPath:
540 if output["code"] == "error":
544 clientPath = output.get("path")
545 elif "data" in output:
546 data = output.get("data")
547 lastSpace = data.rfind(" ")
548 clientPath = data[lastSpace + 1:]
550 if clientPath.endswith("..."):
551 clientPath = clientPath[:-3]
554 def currentGitBranch():
555 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
557 def isValidGitDir(path):
558 if (os.path.exists(path + "/HEAD")
559 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
563 def parseRevision(ref):
564 return read_pipe("git rev-parse %s" % ref).strip()
566 def branchExists(ref):
567 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
571 def extractLogMessageFromGitCommit(commit):
574 ## fixme: title is first line of commit, not 1st paragraph.
576 for log in read_pipe_lines("git cat-file commit %s" % commit):
585 def extractSettingsGitLog(log):
587 for line in log.split("\n"):
589 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
593 assignments = m.group(1).split (':')
594 for a in assignments:
596 key = vals[0].strip()
597 val = ('='.join (vals[1:])).strip()
598 if val.endswith ('\"') and val.startswith('"'):
603 paths = values.get("depot-paths")
605 paths = values.get("depot-path")
607 values['depot-paths'] = paths.split(',')
610 def gitBranchExists(branch):
611 proc = subprocess.Popen(["git", "rev-parse", branch],
612 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
613 return proc.wait() == 0;
617 def gitConfig(key, typeSpecifier=None):
618 if not _gitConfig.has_key(key):
619 cmd = [ "git", "config" ]
621 cmd += [ typeSpecifier ]
623 s = read_pipe(cmd, ignore_error=True)
624 _gitConfig[key] = s.strip()
625 return _gitConfig[key]
627 def gitConfigBool(key):
628 """Return a bool, using git config --bool. It is True only if the
629 variable is set to true, and False if set to false or not present
632 if not _gitConfig.has_key(key):
633 _gitConfig[key] = gitConfig(key, '--bool') == "true"
634 return _gitConfig[key]
636 def gitConfigInt(key):
637 if not _gitConfig.has_key(key):
638 cmd = [ "git", "config", "--int", key ]
639 s = read_pipe(cmd, ignore_error=True)
642 _gitConfig[key] = int(gitConfig(key, '--int'))
644 _gitConfig[key] = None
645 return _gitConfig[key]
647 def gitConfigList(key):
648 if not _gitConfig.has_key(key):
649 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
650 _gitConfig[key] = s.strip().split(os.linesep)
651 if _gitConfig[key] == ['']:
653 return _gitConfig[key]
655 def p4BranchesInGit(branchesAreInRemotes=True):
656 """Find all the branches whose names start with "p4/", looking
657 in remotes or heads as specified by the argument. Return
658 a dictionary of { branch: revision } for each one found.
659 The branch names are the short names, without any
664 cmdline = "git rev-parse --symbolic "
665 if branchesAreInRemotes:
666 cmdline += "--remotes"
668 cmdline += "--branches"
670 for line in read_pipe_lines(cmdline):
674 if not line.startswith('p4/'):
676 # special symbolic ref to p4/master
677 if line == "p4/HEAD":
680 # strip off p4/ prefix
681 branch = line[len("p4/"):]
683 branches[branch] = parseRevision(line)
687 def branch_exists(branch):
688 """Make sure that the given ref name really exists."""
690 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
691 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
692 out, _ = p.communicate()
695 # expect exactly one line of output: the branch name
696 return out.rstrip() == branch
698 def findUpstreamBranchPoint(head = "HEAD"):
699 branches = p4BranchesInGit()
700 # map from depot-path to branch name
701 branchByDepotPath = {}
702 for branch in branches.keys():
703 tip = branches[branch]
704 log = extractLogMessageFromGitCommit(tip)
705 settings = extractSettingsGitLog(log)
706 if settings.has_key("depot-paths"):
707 paths = ",".join(settings["depot-paths"])
708 branchByDepotPath[paths] = "remotes/p4/" + branch
712 while parent < 65535:
713 commit = head + "~%s" % parent
714 log = extractLogMessageFromGitCommit(commit)
715 settings = extractSettingsGitLog(log)
716 if settings.has_key("depot-paths"):
717 paths = ",".join(settings["depot-paths"])
718 if branchByDepotPath.has_key(paths):
719 return [branchByDepotPath[paths], settings]
723 return ["", settings]
725 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
727 print ("Creating/updating branch(es) in %s based on origin branch(es)"
730 originPrefix = "origin/p4/"
732 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
734 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
737 headName = line[len(originPrefix):]
738 remoteHead = localRefPrefix + headName
741 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
742 if (not original.has_key('depot-paths')
743 or not original.has_key('change')):
747 if not gitBranchExists(remoteHead):
749 print "creating %s" % remoteHead
752 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
753 if settings.has_key('change') > 0:
754 if settings['depot-paths'] == original['depot-paths']:
755 originP4Change = int(original['change'])
756 p4Change = int(settings['change'])
757 if originP4Change > p4Change:
758 print ("%s (%s) is newer than %s (%s). "
759 "Updating p4 branch from origin."
760 % (originHead, originP4Change,
761 remoteHead, p4Change))
764 print ("Ignoring: %s was imported from %s while "
765 "%s was imported from %s"
766 % (originHead, ','.join(original['depot-paths']),
767 remoteHead, ','.join(settings['depot-paths'])))
770 system("git update-ref %s %s" % (remoteHead, originHead))
772 def originP4BranchesExist():
773 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
776 def p4ParseNumericChangeRange(parts):
777 changeStart = int(parts[0][1:])
778 if parts[1] == '#head':
779 changeEnd = p4_last_change()
781 changeEnd = int(parts[1])
783 return (changeStart, changeEnd)
785 def chooseBlockSize(blockSize):
789 return defaultBlockSize
791 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
794 # Parse the change range into start and end. Try to find integer
795 # revision ranges as these can be broken up into blocks to avoid
796 # hitting server-side limits (maxrows, maxscanresults). But if
797 # that doesn't work, fall back to using the raw revision specifier
798 # strings, without using block mode.
800 if changeRange is None or changeRange == '':
802 changeEnd = p4_last_change()
803 block_size = chooseBlockSize(requestedBlockSize)
805 parts = changeRange.split(',')
806 assert len(parts) == 2
808 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
809 block_size = chooseBlockSize(requestedBlockSize)
811 changeStart = parts[0][1:]
813 if requestedBlockSize:
814 die("cannot use --changes-block-size with non-numeric revisions")
817 # Accumulate change numbers in a dictionary to avoid duplicates
821 # Retrieve changes a block at a time, to prevent running
822 # into a MaxResults/MaxScanRows error from the server.
828 end = min(changeEnd, changeStart + block_size)
829 revisionRange = "%d,%d" % (changeStart, end)
831 revisionRange = "%s,%s" % (changeStart, changeEnd)
833 cmd += ["%s...@%s" % (p, revisionRange)]
835 for line in p4_read_pipe_lines(cmd):
836 changeNum = int(line.split(" ")[1])
837 changes[changeNum] = True
845 changeStart = end + 1
847 changelist = changes.keys()
851 def p4PathStartsWith(path, prefix):
852 # This method tries to remedy a potential mixed-case issue:
854 # If UserA adds //depot/DirA/file1
855 # and UserB adds //depot/dira/file2
857 # we may or may not have a problem. If you have core.ignorecase=true,
858 # we treat DirA and dira as the same directory
859 if gitConfigBool("core.ignorecase"):
860 return path.lower().startswith(prefix.lower())
861 return path.startswith(prefix)
864 """Look at the p4 client spec, create a View() object that contains
865 all the mappings, and return it."""
867 specList = p4CmdList("client -o")
868 if len(specList) != 1:
869 die('Output from "client -o" is %d lines, expecting 1' %
872 # dictionary of all client parameters
876 client_name = entry["Client"]
878 # just the keys that start with "View"
879 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
882 view = View(client_name)
884 # append the lines, in order, to the view
885 for view_num in range(len(view_keys)):
886 k = "View%d" % view_num
887 if k not in view_keys:
888 die("Expected view key %s missing" % k)
889 view.append(entry[k])
894 """Grab the client directory."""
896 output = p4CmdList("client -o")
898 die('Output from "client -o" is %d lines, expecting 1' % len(output))
901 if "Root" not in entry:
902 die('Client has no "Root"')
907 # P4 wildcards are not allowed in filenames. P4 complains
908 # if you simply add them, but you can force it with "-f", in
909 # which case it translates them into %xx encoding internally.
911 def wildcard_decode(path):
912 # Search for and fix just these four characters. Do % last so
913 # that fixing it does not inadvertently create new %-escapes.
914 # Cannot have * in a filename in windows; untested as to
915 # what p4 would do in such a case.
916 if not platform.system() == "Windows":
917 path = path.replace("%2A", "*")
918 path = path.replace("%23", "#") \
919 .replace("%40", "@") \
923 def wildcard_encode(path):
924 # do % first to avoid double-encoding the %s introduced here
925 path = path.replace("%", "%25") \
926 .replace("*", "%2A") \
927 .replace("#", "%23") \
931 def wildcard_present(path):
932 m = re.search("[*#@%]", path)
935 class LargeFileSystem(object):
936 """Base class for large file system support."""
938 def __init__(self, writeToGitStream):
939 self.largeFiles = set()
940 self.writeToGitStream = writeToGitStream
942 def generatePointer(self, cloneDestination, contentFile):
943 """Return the content of a pointer file that is stored in Git instead of
944 the actual content."""
945 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
947 def pushFile(self, localLargeFile):
948 """Push the actual content which is not stored in the Git repository to
950 assert False, "Method 'pushFile' required in " + self.__class__.__name__
952 def hasLargeFileExtension(self, relPath):
955 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
959 def generateTempFile(self, contents):
960 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
964 return contentFile.name
966 def exceedsLargeFileThreshold(self, relPath, contents):
967 if gitConfigInt('git-p4.largeFileThreshold'):
968 contentsSize = sum(len(d) for d in contents)
969 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
971 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
972 contentsSize = sum(len(d) for d in contents)
973 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
975 contentTempFile = self.generateTempFile(contents)
976 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
977 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
978 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
980 compressedContentsSize = zf.infolist()[0].compress_size
981 os.remove(contentTempFile)
982 os.remove(compressedContentFile.name)
983 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
987 def addLargeFile(self, relPath):
988 self.largeFiles.add(relPath)
990 def removeLargeFile(self, relPath):
991 self.largeFiles.remove(relPath)
993 def isLargeFile(self, relPath):
994 return relPath in self.largeFiles
996 def processContent(self, git_mode, relPath, contents):
997 """Processes the content of git fast import. This method decides if a
998 file is stored in the large file system and handles all necessary
1000 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1001 contentTempFile = self.generateTempFile(contents)
1002 (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1004 # Move temp file to final location in large file system
1005 largeFileDir = os.path.dirname(localLargeFile)
1006 if not os.path.isdir(largeFileDir):
1007 os.makedirs(largeFileDir)
1008 shutil.move(contentTempFile, localLargeFile)
1009 self.addLargeFile(relPath)
1010 if gitConfigBool('git-p4.largeFilePush'):
1011 self.pushFile(localLargeFile)
1013 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1014 return (git_mode, contents)
1016 class MockLFS(LargeFileSystem):
1017 """Mock large file system for testing."""
1019 def generatePointer(self, contentFile):
1020 """The pointer content is the original content prefixed with "pointer-".
1021 The local filename of the large file storage is derived from the file content.
1023 with open(contentFile, 'r') as f:
1026 pointerContents = 'pointer-' + content
1027 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1028 return (gitMode, pointerContents, localLargeFile)
1030 def pushFile(self, localLargeFile):
1031 """The remote filename of the large file storage is the same as the local
1032 one but in a different directory.
1034 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1035 if not os.path.exists(remotePath):
1036 os.makedirs(remotePath)
1037 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1039 class GitLFS(LargeFileSystem):
1040 """Git LFS as backend for the git-p4 large file system.
1041 See https://git-lfs.github.com/ for details."""
1043 def __init__(self, *args):
1044 LargeFileSystem.__init__(self, *args)
1045 self.baseGitAttributes = []
1047 def generatePointer(self, contentFile):
1048 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1049 mode and content which is stored in the Git repository instead of
1050 the actual content. Return also the new location of the actual
1053 pointerProcess = subprocess.Popen(
1054 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1055 stdout=subprocess.PIPE
1057 pointerFile = pointerProcess.stdout.read()
1058 if pointerProcess.wait():
1059 os.remove(contentFile)
1060 die('git-lfs pointer command failed. Did you install the extension?')
1061 pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
1062 oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
1063 localLargeFile = os.path.join(
1065 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1068 # LFS Spec states that pointer files should not have the executable bit set.
1070 return (gitMode, pointerContents, localLargeFile)
1072 def pushFile(self, localLargeFile):
1073 uploadProcess = subprocess.Popen(
1074 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1076 if uploadProcess.wait():
1077 die('git-lfs push command failed. Did you define a remote?')
1079 def generateGitAttributes(self):
1081 self.baseGitAttributes +
1085 '# Git LFS (see https://git-lfs.github.com/)\n',
1088 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1089 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1091 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1092 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1096 def addLargeFile(self, relPath):
1097 LargeFileSystem.addLargeFile(self, relPath)
1098 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1100 def removeLargeFile(self, relPath):
1101 LargeFileSystem.removeLargeFile(self, relPath)
1102 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1104 def processContent(self, git_mode, relPath, contents):
1105 if relPath == '.gitattributes':
1106 self.baseGitAttributes = contents
1107 return (git_mode, self.generateGitAttributes())
1109 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1113 self.usage = "usage: %prog [options]"
1114 self.needsGit = True
1115 self.verbose = False
1119 self.userMapFromPerforceServer = False
1120 self.myP4UserId = None
1124 return self.myP4UserId
1126 results = p4CmdList("user -o")
1128 if r.has_key('User'):
1129 self.myP4UserId = r['User']
1131 die("Could not find your p4 user id")
1133 def p4UserIsMe(self, p4User):
1134 # return True if the given p4 user is actually me
1135 me = self.p4UserId()
1136 if not p4User or p4User != me:
1141 def getUserCacheFilename(self):
1142 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1143 return home + "/.gitp4-usercache.txt"
1145 def getUserMapFromPerforceServer(self):
1146 if self.userMapFromPerforceServer:
1151 for output in p4CmdList("users"):
1152 if not output.has_key("User"):
1154 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1155 self.emails[output["Email"]] = output["User"]
1159 for (key, val) in self.users.items():
1160 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1162 open(self.getUserCacheFilename(), "wb").write(s)
1163 self.userMapFromPerforceServer = True
1165 def loadUserMapFromCache(self):
1167 self.userMapFromPerforceServer = False
1169 cache = open(self.getUserCacheFilename(), "rb")
1170 lines = cache.readlines()
1173 entry = line.strip().split("\t")
1174 self.users[entry[0]] = entry[1]
1176 self.getUserMapFromPerforceServer()
1178 class P4Debug(Command):
1180 Command.__init__(self)
1182 self.description = "A tool to debug the output of p4 -G."
1183 self.needsGit = False
1185 def run(self, args):
1187 for output in p4CmdList(args):
1188 print 'Element: %d' % j
1193 class P4RollBack(Command):
1195 Command.__init__(self)
1197 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1199 self.description = "A tool to debug the multi-branch import. Don't use :)"
1200 self.rollbackLocalBranches = False
1202 def run(self, args):
1205 maxChange = int(args[0])
1207 if "p4ExitCode" in p4Cmd("changes -m 1"):
1208 die("Problems executing p4");
1210 if self.rollbackLocalBranches:
1211 refPrefix = "refs/heads/"
1212 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1214 refPrefix = "refs/remotes/"
1215 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1218 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1220 ref = refPrefix + line
1221 log = extractLogMessageFromGitCommit(ref)
1222 settings = extractSettingsGitLog(log)
1224 depotPaths = settings['depot-paths']
1225 change = settings['change']
1229 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1230 for p in depotPaths]))) == 0:
1231 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1232 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1235 while change and int(change) > maxChange:
1238 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1239 system("git update-ref %s \"%s^\"" % (ref, ref))
1240 log = extractLogMessageFromGitCommit(ref)
1241 settings = extractSettingsGitLog(log)
1244 depotPaths = settings['depot-paths']
1245 change = settings['change']
1248 print "%s rewound to %s" % (ref, change)
1252 class P4Submit(Command, P4UserMap):
1254 conflict_behavior_choices = ("ask", "skip", "quit")
1257 Command.__init__(self)
1258 P4UserMap.__init__(self)
1260 optparse.make_option("--origin", dest="origin"),
1261 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1262 # preserve the user, requires relevant p4 permissions
1263 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1264 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1265 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1266 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1267 optparse.make_option("--conflict", dest="conflict_behavior",
1268 choices=self.conflict_behavior_choices),
1269 optparse.make_option("--branch", dest="branch"),
1271 self.description = "Submit changes from git to the perforce depot."
1272 self.usage += " [name of git branch to submit into perforce depot]"
1274 self.detectRenames = False
1275 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1276 self.dry_run = False
1277 self.prepare_p4_only = False
1278 self.conflict_behavior = None
1279 self.isWindows = (platform.system() == "Windows")
1280 self.exportLabels = False
1281 self.p4HasMoveCommand = p4_has_move_command()
1284 if gitConfig('git-p4.largeFileSystem'):
1285 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1288 if len(p4CmdList("opened ...")) > 0:
1289 die("You have files opened with perforce! Close them before starting the sync.")
1291 def separate_jobs_from_description(self, message):
1292 """Extract and return a possible Jobs field in the commit
1293 message. It goes into a separate section in the p4 change
1296 A jobs line starts with "Jobs:" and looks like a new field
1297 in a form. Values are white-space separated on the same
1298 line or on following lines that start with a tab.
1300 This does not parse and extract the full git commit message
1301 like a p4 form. It just sees the Jobs: line as a marker
1302 to pass everything from then on directly into the p4 form,
1303 but outside the description section.
1305 Return a tuple (stripped log message, jobs string)."""
1307 m = re.search(r'^Jobs:', message, re.MULTILINE)
1309 return (message, None)
1311 jobtext = message[m.start():]
1312 stripped_message = message[:m.start()].rstrip()
1313 return (stripped_message, jobtext)
1315 def prepareLogMessage(self, template, message, jobs):
1316 """Edits the template returned from "p4 change -o" to insert
1317 the message in the Description field, and the jobs text in
1321 inDescriptionSection = False
1323 for line in template.split("\n"):
1324 if line.startswith("#"):
1325 result += line + "\n"
1328 if inDescriptionSection:
1329 if line.startswith("Files:") or line.startswith("Jobs:"):
1330 inDescriptionSection = False
1331 # insert Jobs section
1333 result += jobs + "\n"
1337 if line.startswith("Description:"):
1338 inDescriptionSection = True
1340 for messageLine in message.split("\n"):
1341 line += "\t" + messageLine + "\n"
1343 result += line + "\n"
1347 def patchRCSKeywords(self, file, pattern):
1348 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1349 (handle, outFileName) = tempfile.mkstemp(dir='.')
1351 outFile = os.fdopen(handle, "w+")
1352 inFile = open(file, "r")
1353 regexp = re.compile(pattern, re.VERBOSE)
1354 for line in inFile.readlines():
1355 line = regexp.sub(r'$\1$', line)
1359 # Forcibly overwrite the original file
1361 shutil.move(outFileName, file)
1363 # cleanup our temporary file
1364 os.unlink(outFileName)
1365 print "Failed to strip RCS keywords in %s" % file
1368 print "Patched up RCS keywords in %s" % file
1370 def p4UserForCommit(self,id):
1371 # Return the tuple (perforce user,git email) for a given git commit id
1372 self.getUserMapFromPerforceServer()
1373 gitEmail = read_pipe(["git", "log", "--max-count=1",
1374 "--format=%ae", id])
1375 gitEmail = gitEmail.strip()
1376 if not self.emails.has_key(gitEmail):
1377 return (None,gitEmail)
1379 return (self.emails[gitEmail],gitEmail)
1381 def checkValidP4Users(self,commits):
1382 # check if any git authors cannot be mapped to p4 users
1384 (user,email) = self.p4UserForCommit(id)
1386 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1387 if gitConfigBool("git-p4.allowMissingP4Users"):
1390 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1392 def lastP4Changelist(self):
1393 # Get back the last changelist number submitted in this client spec. This
1394 # then gets used to patch up the username in the change. If the same
1395 # client spec is being used by multiple processes then this might go
1397 results = p4CmdList("client -o") # find the current client
1400 if r.has_key('Client'):
1401 client = r['Client']
1404 die("could not get client spec")
1405 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1407 if r.has_key('change'):
1409 die("Could not get changelist number for last submit - cannot patch up user details")
1411 def modifyChangelistUser(self, changelist, newUser):
1412 # fixup the user field of a changelist after it has been submitted.
1413 changes = p4CmdList("change -o %s" % changelist)
1414 if len(changes) != 1:
1415 die("Bad output from p4 change modifying %s to user %s" %
1416 (changelist, newUser))
1419 if c['User'] == newUser: return # nothing to do
1421 input = marshal.dumps(c)
1423 result = p4CmdList("change -f -i", stdin=input)
1425 if r.has_key('code'):
1426 if r['code'] == 'error':
1427 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1428 if r.has_key('data'):
1429 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1431 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1433 def canChangeChangelists(self):
1434 # check to see if we have p4 admin or super-user permissions, either of
1435 # which are required to modify changelists.
1436 results = p4CmdList(["protects", self.depotPath])
1438 if r.has_key('perm'):
1439 if r['perm'] == 'admin':
1441 if r['perm'] == 'super':
1445 def prepareSubmitTemplate(self):
1446 """Run "p4 change -o" to grab a change specification template.
1447 This does not use "p4 -G", as it is nice to keep the submission
1448 template in original order, since a human might edit it.
1450 Remove lines in the Files section that show changes to files
1451 outside the depot path we're committing into."""
1454 inFilesSection = False
1455 for line in p4_read_pipe_lines(['change', '-o']):
1456 if line.endswith("\r\n"):
1457 line = line[:-2] + "\n"
1459 if line.startswith("\t"):
1460 # path starts and ends with a tab
1462 lastTab = path.rfind("\t")
1464 path = path[:lastTab]
1465 if not p4PathStartsWith(path, self.depotPath):
1468 inFilesSection = False
1470 if line.startswith("Files:"):
1471 inFilesSection = True
1477 def edit_template(self, template_file):
1478 """Invoke the editor to let the user change the submission
1479 message. Return true if okay to continue with the submit."""
1481 # if configured to skip the editing part, just submit
1482 if gitConfigBool("git-p4.skipSubmitEdit"):
1485 # look at the modification time, to check later if the user saved
1487 mtime = os.stat(template_file).st_mtime
1490 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1491 editor = os.environ.get("P4EDITOR")
1493 editor = read_pipe("git var GIT_EDITOR").strip()
1494 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1496 # If the file was not saved, prompt to see if this patch should
1497 # be skipped. But skip this verification step if configured so.
1498 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1501 # modification time updated means user saved the file
1502 if os.stat(template_file).st_mtime > mtime:
1506 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1512 def get_diff_description(self, editedFiles, filesToAdd):
1514 if os.environ.has_key("P4DIFF"):
1515 del(os.environ["P4DIFF"])
1517 for editedFile in editedFiles:
1518 diff += p4_read_pipe(['diff', '-du',
1519 wildcard_encode(editedFile)])
1523 for newFile in filesToAdd:
1524 newdiff += "==== new file ====\n"
1525 newdiff += "--- /dev/null\n"
1526 newdiff += "+++ %s\n" % newFile
1527 f = open(newFile, "r")
1528 for line in f.readlines():
1529 newdiff += "+" + line
1532 return (diff + newdiff).replace('\r\n', '\n')
1534 def applyCommit(self, id):
1535 """Apply one commit, return True if it succeeded."""
1537 print "Applying", read_pipe(["git", "show", "-s",
1538 "--format=format:%h %s", id])
1540 (p4User, gitEmail) = self.p4UserForCommit(id)
1542 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1544 filesToDelete = set()
1546 pureRenameCopy = set()
1547 filesToChangeExecBit = {}
1550 diff = parseDiffTreeEntry(line)
1551 modifier = diff['status']
1555 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1556 filesToChangeExecBit[path] = diff['dst_mode']
1557 editedFiles.add(path)
1558 elif modifier == "A":
1559 filesToAdd.add(path)
1560 filesToChangeExecBit[path] = diff['dst_mode']
1561 if path in filesToDelete:
1562 filesToDelete.remove(path)
1563 elif modifier == "D":
1564 filesToDelete.add(path)
1565 if path in filesToAdd:
1566 filesToAdd.remove(path)
1567 elif modifier == "C":
1568 src, dest = diff['src'], diff['dst']
1569 p4_integrate(src, dest)
1570 pureRenameCopy.add(dest)
1571 if diff['src_sha1'] != diff['dst_sha1']:
1573 pureRenameCopy.discard(dest)
1574 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1576 pureRenameCopy.discard(dest)
1577 filesToChangeExecBit[dest] = diff['dst_mode']
1579 # turn off read-only attribute
1580 os.chmod(dest, stat.S_IWRITE)
1582 editedFiles.add(dest)
1583 elif modifier == "R":
1584 src, dest = diff['src'], diff['dst']
1585 if self.p4HasMoveCommand:
1586 p4_edit(src) # src must be open before move
1587 p4_move(src, dest) # opens for (move/delete, move/add)
1589 p4_integrate(src, dest)
1590 if diff['src_sha1'] != diff['dst_sha1']:
1593 pureRenameCopy.add(dest)
1594 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1595 if not self.p4HasMoveCommand:
1596 p4_edit(dest) # with move: already open, writable
1597 filesToChangeExecBit[dest] = diff['dst_mode']
1598 if not self.p4HasMoveCommand:
1600 os.chmod(dest, stat.S_IWRITE)
1602 filesToDelete.add(src)
1603 editedFiles.add(dest)
1605 die("unknown modifier %s for %s" % (modifier, path))
1607 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1608 patchcmd = diffcmd + " | git apply "
1609 tryPatchCmd = patchcmd + "--check -"
1610 applyPatchCmd = patchcmd + "--check --apply -"
1611 patch_succeeded = True
1613 if os.system(tryPatchCmd) != 0:
1614 fixed_rcs_keywords = False
1615 patch_succeeded = False
1616 print "Unfortunately applying the change failed!"
1618 # Patch failed, maybe it's just RCS keyword woes. Look through
1619 # the patch to see if that's possible.
1620 if gitConfigBool("git-p4.attemptRCSCleanup"):
1624 for file in editedFiles | filesToDelete:
1625 # did this file's delta contain RCS keywords?
1626 pattern = p4_keywords_regexp_for_file(file)
1629 # this file is a possibility...look for RCS keywords.
1630 regexp = re.compile(pattern, re.VERBOSE)
1631 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1632 if regexp.search(line):
1634 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1635 kwfiles[file] = pattern
1638 for file in kwfiles:
1640 print "zapping %s with %s" % (line,pattern)
1641 # File is being deleted, so not open in p4. Must
1642 # disable the read-only bit on windows.
1643 if self.isWindows and file not in editedFiles:
1644 os.chmod(file, stat.S_IWRITE)
1645 self.patchRCSKeywords(file, kwfiles[file])
1646 fixed_rcs_keywords = True
1648 if fixed_rcs_keywords:
1649 print "Retrying the patch with RCS keywords cleaned up"
1650 if os.system(tryPatchCmd) == 0:
1651 patch_succeeded = True
1653 if not patch_succeeded:
1654 for f in editedFiles:
1659 # Apply the patch for real, and do add/delete/+x handling.
1661 system(applyPatchCmd)
1663 for f in filesToAdd:
1665 for f in filesToDelete:
1669 # Set/clear executable bits
1670 for f in filesToChangeExecBit.keys():
1671 mode = filesToChangeExecBit[f]
1672 setP4ExecBit(f, mode)
1675 # Build p4 change description, starting with the contents
1676 # of the git commit message.
1678 logMessage = extractLogMessageFromGitCommit(id)
1679 logMessage = logMessage.strip()
1680 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1682 template = self.prepareSubmitTemplate()
1683 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1685 if self.preserveUser:
1686 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1688 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1689 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1690 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1691 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1693 separatorLine = "######## everything below this line is just the diff #######\n"
1694 if not self.prepare_p4_only:
1695 submitTemplate += separatorLine
1696 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1698 (handle, fileName) = tempfile.mkstemp()
1699 tmpFile = os.fdopen(handle, "w+b")
1701 submitTemplate = submitTemplate.replace("\n", "\r\n")
1702 tmpFile.write(submitTemplate)
1705 if self.prepare_p4_only:
1707 # Leave the p4 tree prepared, and the submit template around
1708 # and let the user decide what to do next
1711 print "P4 workspace prepared for submission."
1712 print "To submit or revert, go to client workspace"
1713 print " " + self.clientPath
1715 print "To submit, use \"p4 submit\" to write a new description,"
1716 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1717 " \"git p4\"." % fileName
1718 print "You can delete the file \"%s\" when finished." % fileName
1720 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1721 print "To preserve change ownership by user %s, you must\n" \
1722 "do \"p4 change -f <change>\" after submitting and\n" \
1723 "edit the User field."
1725 print "After submitting, renamed files must be re-synced."
1726 print "Invoke \"p4 sync -f\" on each of these files:"
1727 for f in pureRenameCopy:
1731 print "To revert the changes, use \"p4 revert ...\", and delete"
1732 print "the submit template file \"%s\"" % fileName
1734 print "Since the commit adds new files, they must be deleted:"
1735 for f in filesToAdd:
1741 # Let the user edit the change description, then submit it.
1743 if self.edit_template(fileName):
1744 # read the edited message and submit
1746 tmpFile = open(fileName, "rb")
1747 message = tmpFile.read()
1750 message = message.replace("\r\n", "\n")
1751 submitTemplate = message[:message.index(separatorLine)]
1752 p4_write_pipe(['submit', '-i'], submitTemplate)
1754 if self.preserveUser:
1756 # Get last changelist number. Cannot easily get it from
1757 # the submit command output as the output is
1759 changelist = self.lastP4Changelist()
1760 self.modifyChangelistUser(changelist, p4User)
1762 # The rename/copy happened by applying a patch that created a
1763 # new file. This leaves it writable, which confuses p4.
1764 for f in pureRenameCopy:
1770 print "Submission cancelled, undoing p4 changes."
1771 for f in editedFiles:
1773 for f in filesToAdd:
1776 for f in filesToDelete:
1782 # Export git tags as p4 labels. Create a p4 label and then tag
1784 def exportGitTags(self, gitTags):
1785 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1786 if len(validLabelRegexp) == 0:
1787 validLabelRegexp = defaultLabelRegexp
1788 m = re.compile(validLabelRegexp)
1790 for name in gitTags:
1792 if not m.match(name):
1794 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1797 # Get the p4 commit this corresponds to
1798 logMessage = extractLogMessageFromGitCommit(name)
1799 values = extractSettingsGitLog(logMessage)
1801 if not values.has_key('change'):
1802 # a tag pointing to something not sent to p4; ignore
1804 print "git tag %s does not give a p4 commit" % name
1807 changelist = values['change']
1809 # Get the tag details.
1813 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1816 if re.match(r'tag\s+', l):
1818 elif re.match(r'\s*$', l):
1825 body = ["lightweight tag imported by git p4\n"]
1827 # Create the label - use the same view as the client spec we are using
1828 clientSpec = getClientSpec()
1830 labelTemplate = "Label: %s\n" % name
1831 labelTemplate += "Description:\n"
1833 labelTemplate += "\t" + b + "\n"
1834 labelTemplate += "View:\n"
1835 for depot_side in clientSpec.mappings:
1836 labelTemplate += "\t%s\n" % depot_side
1839 print "Would create p4 label %s for tag" % name
1840 elif self.prepare_p4_only:
1841 print "Not creating p4 label %s for tag due to option" \
1842 " --prepare-p4-only" % name
1844 p4_write_pipe(["label", "-i"], labelTemplate)
1847 p4_system(["tag", "-l", name] +
1848 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1851 print "created p4 label for tag %s" % name
1853 def run(self, args):
1855 self.master = currentGitBranch()
1856 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1857 die("Detecting current git branch failed!")
1858 elif len(args) == 1:
1859 self.master = args[0]
1860 if not branchExists(self.master):
1861 die("Branch %s does not exist" % self.master)
1865 allowSubmit = gitConfig("git-p4.allowSubmit")
1866 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1867 die("%s is not in git-p4.allowSubmit" % self.master)
1869 [upstream, settings] = findUpstreamBranchPoint()
1870 self.depotPath = settings['depot-paths'][0]
1871 if len(self.origin) == 0:
1872 self.origin = upstream
1874 if self.preserveUser:
1875 if not self.canChangeChangelists():
1876 die("Cannot preserve user names without p4 super-user or admin permissions")
1878 # if not set from the command line, try the config file
1879 if self.conflict_behavior is None:
1880 val = gitConfig("git-p4.conflict")
1882 if val not in self.conflict_behavior_choices:
1883 die("Invalid value '%s' for config git-p4.conflict" % val)
1886 self.conflict_behavior = val
1889 print "Origin branch is " + self.origin
1891 if len(self.depotPath) == 0:
1892 print "Internal error: cannot locate perforce depot path from existing branches"
1895 self.useClientSpec = False
1896 if gitConfigBool("git-p4.useclientspec"):
1897 self.useClientSpec = True
1898 if self.useClientSpec:
1899 self.clientSpecDirs = getClientSpec()
1901 # Check for the existance of P4 branches
1902 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1904 if self.useClientSpec and not branchesDetected:
1905 # all files are relative to the client spec
1906 self.clientPath = getClientRoot()
1908 self.clientPath = p4Where(self.depotPath)
1910 if self.clientPath == "":
1911 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1913 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1914 self.oldWorkingDirectory = os.getcwd()
1916 # ensure the clientPath exists
1917 new_client_dir = False
1918 if not os.path.exists(self.clientPath):
1919 new_client_dir = True
1920 os.makedirs(self.clientPath)
1922 chdir(self.clientPath, is_client_path=True)
1924 print "Would synchronize p4 checkout in %s" % self.clientPath
1926 print "Synchronizing p4 checkout..."
1928 # old one was destroyed, and maybe nobody told p4
1929 p4_sync("...", "-f")
1935 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1936 commits.append(line.strip())
1939 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1940 self.checkAuthorship = False
1942 self.checkAuthorship = True
1944 if self.preserveUser:
1945 self.checkValidP4Users(commits)
1948 # Build up a set of options to be passed to diff when
1949 # submitting each commit to p4.
1951 if self.detectRenames:
1952 # command-line -M arg
1953 self.diffOpts = "-M"
1955 # If not explicitly set check the config variable
1956 detectRenames = gitConfig("git-p4.detectRenames")
1958 if detectRenames.lower() == "false" or detectRenames == "":
1960 elif detectRenames.lower() == "true":
1961 self.diffOpts = "-M"
1963 self.diffOpts = "-M%s" % detectRenames
1965 # no command-line arg for -C or --find-copies-harder, just
1967 detectCopies = gitConfig("git-p4.detectCopies")
1968 if detectCopies.lower() == "false" or detectCopies == "":
1970 elif detectCopies.lower() == "true":
1971 self.diffOpts += " -C"
1973 self.diffOpts += " -C%s" % detectCopies
1975 if gitConfigBool("git-p4.detectCopiesHarder"):
1976 self.diffOpts += " --find-copies-harder"
1979 # Apply the commits, one at a time. On failure, ask if should
1980 # continue to try the rest of the patches, or quit.
1985 last = len(commits) - 1
1986 for i, commit in enumerate(commits):
1988 print " ", read_pipe(["git", "show", "-s",
1989 "--format=format:%h %s", commit])
1992 ok = self.applyCommit(commit)
1994 applied.append(commit)
1996 if self.prepare_p4_only and i < last:
1997 print "Processing only the first commit due to option" \
1998 " --prepare-p4-only"
2003 # prompt for what to do, or use the option/variable
2004 if self.conflict_behavior == "ask":
2005 print "What do you want to do?"
2006 response = raw_input("[s]kip this commit but apply"
2007 " the rest, or [q]uit? ")
2010 elif self.conflict_behavior == "skip":
2012 elif self.conflict_behavior == "quit":
2015 die("Unknown conflict_behavior '%s'" %
2016 self.conflict_behavior)
2018 if response[0] == "s":
2019 print "Skipping this commit, but applying the rest"
2021 if response[0] == "q":
2028 chdir(self.oldWorkingDirectory)
2032 elif self.prepare_p4_only:
2034 elif len(commits) == len(applied):
2035 print "All commits applied!"
2039 sync.branch = self.branch
2046 if len(applied) == 0:
2047 print "No commits applied."
2049 print "Applied only the commits marked with '*':"
2055 print star, read_pipe(["git", "show", "-s",
2056 "--format=format:%h %s", c])
2057 print "You will have to do 'git p4 sync' and rebase."
2059 if gitConfigBool("git-p4.exportLabels"):
2060 self.exportLabels = True
2062 if self.exportLabels:
2063 p4Labels = getP4Labels(self.depotPath)
2064 gitTags = getGitTags()
2066 missingGitTags = gitTags - p4Labels
2067 self.exportGitTags(missingGitTags)
2069 # exit with error unless everything applied perfectly
2070 if len(commits) != len(applied):
2076 """Represent a p4 view ("p4 help views"), and map files in a
2077 repo according to the view."""
2079 def __init__(self, client_name):
2081 self.client_prefix = "//%s/" % client_name
2082 # cache results of "p4 where" to lookup client file locations
2083 self.client_spec_path_cache = {}
2085 def append(self, view_line):
2086 """Parse a view line, splitting it into depot and client
2087 sides. Append to self.mappings, preserving order. This
2088 is only needed for tag creation."""
2090 # Split the view line into exactly two words. P4 enforces
2091 # structure on these lines that simplifies this quite a bit.
2093 # Either or both words may be double-quoted.
2094 # Single quotes do not matter.
2095 # Double-quote marks cannot occur inside the words.
2096 # A + or - prefix is also inside the quotes.
2097 # There are no quotes unless they contain a space.
2098 # The line is already white-space stripped.
2099 # The two words are separated by a single space.
2101 if view_line[0] == '"':
2102 # First word is double quoted. Find its end.
2103 close_quote_index = view_line.find('"', 1)
2104 if close_quote_index <= 0:
2105 die("No first-word closing quote found: %s" % view_line)
2106 depot_side = view_line[1:close_quote_index]
2107 # skip closing quote and space
2108 rhs_index = close_quote_index + 1 + 1
2110 space_index = view_line.find(" ")
2111 if space_index <= 0:
2112 die("No word-splitting space found: %s" % view_line)
2113 depot_side = view_line[0:space_index]
2114 rhs_index = space_index + 1
2116 # prefix + means overlay on previous mapping
2117 if depot_side.startswith("+"):
2118 depot_side = depot_side[1:]
2120 # prefix - means exclude this path, leave out of mappings
2122 if depot_side.startswith("-"):
2124 depot_side = depot_side[1:]
2127 self.mappings.append(depot_side)
2129 def convert_client_path(self, clientFile):
2130 # chop off //client/ part to make it relative
2131 if not clientFile.startswith(self.client_prefix):
2132 die("No prefix '%s' on clientFile '%s'" %
2133 (self.client_prefix, clientFile))
2134 return clientFile[len(self.client_prefix):]
2136 def update_client_spec_path_cache(self, files):
2137 """ Caching file paths by "p4 where" batch query """
2139 # List depot file paths exclude that already cached
2140 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2142 if len(fileArgs) == 0:
2143 return # All files in cache
2145 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2146 for res in where_result:
2147 if "code" in res and res["code"] == "error":
2148 # assume error is "... file(s) not in client view"
2150 if "clientFile" not in res:
2151 die("No clientFile in 'p4 where' output")
2153 # it will list all of them, but only one not unmap-ped
2155 if gitConfigBool("core.ignorecase"):
2156 res['depotFile'] = res['depotFile'].lower()
2157 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2159 # not found files or unmap files set to ""
2160 for depotFile in fileArgs:
2161 if gitConfigBool("core.ignorecase"):
2162 depotFile = depotFile.lower()
2163 if depotFile not in self.client_spec_path_cache:
2164 self.client_spec_path_cache[depotFile] = ""
2166 def map_in_client(self, depot_path):
2167 """Return the relative location in the client where this
2168 depot file should live. Returns "" if the file should
2169 not be mapped in the client."""
2171 if gitConfigBool("core.ignorecase"):
2172 depot_path = depot_path.lower()
2174 if depot_path in self.client_spec_path_cache:
2175 return self.client_spec_path_cache[depot_path]
2177 die( "Error: %s is not found in client spec path" % depot_path )
2180 class P4Sync(Command, P4UserMap):
2181 delete_actions = ( "delete", "move/delete", "purge" )
2184 Command.__init__(self)
2185 P4UserMap.__init__(self)
2187 optparse.make_option("--branch", dest="branch"),
2188 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2189 optparse.make_option("--changesfile", dest="changesFile"),
2190 optparse.make_option("--silent", dest="silent", action="store_true"),
2191 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2192 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2193 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2194 help="Import into refs/heads/ , not refs/remotes"),
2195 optparse.make_option("--max-changes", dest="maxChanges",
2196 help="Maximum number of changes to import"),
2197 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2198 help="Internal block size to use when iteratively calling p4 changes"),
2199 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2200 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2201 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2202 help="Only sync files that are included in the Perforce Client Spec"),
2203 optparse.make_option("-/", dest="cloneExclude",
2204 action="append", type="string",
2205 help="exclude depot path"),
2207 self.description = """Imports from Perforce into a git repository.\n
2209 //depot/my/project/ -- to import the current head
2210 //depot/my/project/@all -- to import everything
2211 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2213 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2215 self.usage += " //depot/path[@revRange]"
2217 self.createdBranches = set()
2218 self.committedChanges = set()
2220 self.detectBranches = False
2221 self.detectLabels = False
2222 self.importLabels = False
2223 self.changesFile = ""
2224 self.syncWithOrigin = True
2225 self.importIntoRemotes = True
2226 self.maxChanges = ""
2227 self.changes_block_size = None
2228 self.keepRepoPath = False
2229 self.depotPaths = None
2230 self.p4BranchesInGit = []
2231 self.cloneExclude = []
2232 self.useClientSpec = False
2233 self.useClientSpec_from_options = False
2234 self.clientSpecDirs = None
2235 self.tempBranches = []
2236 self.tempBranchLocation = "git-p4-tmp"
2237 self.largeFileSystem = None
2239 if gitConfig('git-p4.largeFileSystem'):
2240 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2241 self.largeFileSystem = largeFileSystemConstructor(
2242 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2245 if gitConfig("git-p4.syncFromOrigin") == "false":
2246 self.syncWithOrigin = False
2248 # This is required for the "append" cloneExclude action
2249 def ensure_value(self, attr, value):
2250 if not hasattr(self, attr) or getattr(self, attr) is None:
2251 setattr(self, attr, value)
2252 return getattr(self, attr)
2254 # Force a checkpoint in fast-import and wait for it to finish
2255 def checkpoint(self):
2256 self.gitStream.write("checkpoint\n\n")
2257 self.gitStream.write("progress checkpoint\n\n")
2258 out = self.gitOutput.readline()
2260 print "checkpoint finished: " + out
2262 def extractFilesFromCommit(self, commit):
2263 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2264 for path in self.cloneExclude]
2267 while commit.has_key("depotFile%s" % fnum):
2268 path = commit["depotFile%s" % fnum]
2270 if [p for p in self.cloneExclude
2271 if p4PathStartsWith(path, p)]:
2274 found = [p for p in self.depotPaths
2275 if p4PathStartsWith(path, p)]
2282 file["rev"] = commit["rev%s" % fnum]
2283 file["action"] = commit["action%s" % fnum]
2284 file["type"] = commit["type%s" % fnum]
2289 def stripRepoPath(self, path, prefixes):
2290 """When streaming files, this is called to map a p4 depot path
2291 to where it should go in git. The prefixes are either
2292 self.depotPaths, or self.branchPrefixes in the case of
2293 branch detection."""
2295 if self.useClientSpec:
2296 # branch detection moves files up a level (the branch name)
2297 # from what client spec interpretation gives
2298 path = self.clientSpecDirs.map_in_client(path)
2299 if self.detectBranches:
2300 for b in self.knownBranches:
2301 if path.startswith(b + "/"):
2302 path = path[len(b)+1:]
2304 elif self.keepRepoPath:
2305 # Preserve everything in relative path name except leading
2306 # //depot/; just look at first prefix as they all should
2307 # be in the same depot.
2308 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2309 if p4PathStartsWith(path, depot):
2310 path = path[len(depot):]
2314 if p4PathStartsWith(path, p):
2315 path = path[len(p):]
2318 path = wildcard_decode(path)
2321 def splitFilesIntoBranches(self, commit):
2322 """Look at each depotFile in the commit to figure out to what
2323 branch it belongs."""
2325 if self.clientSpecDirs:
2326 files = self.extractFilesFromCommit(commit)
2327 self.clientSpecDirs.update_client_spec_path_cache(files)
2331 while commit.has_key("depotFile%s" % fnum):
2332 path = commit["depotFile%s" % fnum]
2333 found = [p for p in self.depotPaths
2334 if p4PathStartsWith(path, p)]
2341 file["rev"] = commit["rev%s" % fnum]
2342 file["action"] = commit["action%s" % fnum]
2343 file["type"] = commit["type%s" % fnum]
2346 # start with the full relative path where this file would
2348 if self.useClientSpec:
2349 relPath = self.clientSpecDirs.map_in_client(path)
2351 relPath = self.stripRepoPath(path, self.depotPaths)
2353 for branch in self.knownBranches.keys():
2354 # add a trailing slash so that a commit into qt/4.2foo
2355 # doesn't end up in qt/4.2, e.g.
2356 if relPath.startswith(branch + "/"):
2357 if branch not in branches:
2358 branches[branch] = []
2359 branches[branch].append(file)
2364 def writeToGitStream(self, gitMode, relPath, contents):
2365 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2366 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2368 self.gitStream.write(d)
2369 self.gitStream.write('\n')
2371 # output one file from the P4 stream
2372 # - helper for streamP4Files
2374 def streamOneP4File(self, file, contents):
2375 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2377 size = int(self.stream_file['fileSize'])
2378 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2381 (type_base, type_mods) = split_p4_type(file["type"])
2384 if "x" in type_mods:
2386 if type_base == "symlink":
2388 # p4 print on a symlink sometimes contains "target\n";
2389 # if it does, remove the newline
2390 data = ''.join(contents)
2392 # Some version of p4 allowed creating a symlink that pointed
2393 # to nothing. This causes p4 errors when checking out such
2394 # a change, and errors here too. Work around it by ignoring
2395 # the bad symlink; hopefully a future change fixes it.
2396 print "\nIgnoring empty symlink in %s" % file['depotFile']
2398 elif data[-1] == '\n':
2399 contents = [data[:-1]]
2403 if type_base == "utf16":
2404 # p4 delivers different text in the python output to -G
2405 # than it does when using "print -o", or normal p4 client
2406 # operations. utf16 is converted to ascii or utf8, perhaps.
2407 # But ascii text saved as -t utf16 is completely mangled.
2408 # Invoke print -o to get the real contents.
2410 # On windows, the newlines will always be mangled by print, so put
2411 # them back too. This is not needed to the cygwin windows version,
2412 # just the native "NT" type.
2415 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2416 except Exception as e:
2417 if 'Translation of file content failed' in str(e):
2418 type_base = 'binary'
2422 if p4_version_string().find('/NT') >= 0:
2423 text = text.replace('\r\n', '\n')
2426 if type_base == "apple":
2427 # Apple filetype files will be streamed as a concatenation of
2428 # its appledouble header and the contents. This is useless
2429 # on both macs and non-macs. If using "print -q -o xx", it
2430 # will create "xx" with the data, and "%xx" with the header.
2431 # This is also not very useful.
2433 # Ideally, someday, this script can learn how to generate
2434 # appledouble files directly and import those to git, but
2435 # non-mac machines can never find a use for apple filetype.
2436 print "\nIgnoring apple filetype file %s" % file['depotFile']
2439 # Note that we do not try to de-mangle keywords on utf16 files,
2440 # even though in theory somebody may want that.
2441 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2443 regexp = re.compile(pattern, re.VERBOSE)
2444 text = ''.join(contents)
2445 text = regexp.sub(r'$\1$', text)
2449 relPath.decode('ascii')
2452 if gitConfig('git-p4.pathEncoding'):
2453 encoding = gitConfig('git-p4.pathEncoding')
2454 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2456 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2458 if self.largeFileSystem:
2459 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2461 self.writeToGitStream(git_mode, relPath, contents)
2463 def streamOneP4Deletion(self, file):
2464 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2466 sys.stdout.write("delete %s\n" % relPath)
2468 self.gitStream.write("D %s\n" % relPath)
2470 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2471 self.largeFileSystem.removeLargeFile(relPath)
2473 # handle another chunk of streaming data
2474 def streamP4FilesCb(self, marshalled):
2476 # catch p4 errors and complain
2478 if "code" in marshalled:
2479 if marshalled["code"] == "error":
2480 if "data" in marshalled:
2481 err = marshalled["data"].rstrip()
2483 if not err and 'fileSize' in self.stream_file:
2484 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2485 if required_bytes > 0:
2486 err = 'Not enough space left on %s! Free at least %i MB.' % (
2487 os.getcwd(), required_bytes/1024/1024
2492 if self.stream_have_file_info:
2493 if "depotFile" in self.stream_file:
2494 f = self.stream_file["depotFile"]
2495 # force a failure in fast-import, else an empty
2496 # commit will be made
2497 self.gitStream.write("\n")
2498 self.gitStream.write("die-now\n")
2499 self.gitStream.close()
2500 # ignore errors, but make sure it exits first
2501 self.importProcess.wait()
2503 die("Error from p4 print for %s: %s" % (f, err))
2505 die("Error from p4 print: %s" % err)
2507 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2508 # start of a new file - output the old one first
2509 self.streamOneP4File(self.stream_file, self.stream_contents)
2510 self.stream_file = {}
2511 self.stream_contents = []
2512 self.stream_have_file_info = False
2514 # pick up the new file information... for the
2515 # 'data' field we need to append to our array
2516 for k in marshalled.keys():
2518 if 'streamContentSize' not in self.stream_file:
2519 self.stream_file['streamContentSize'] = 0
2520 self.stream_file['streamContentSize'] += len(marshalled['data'])
2521 self.stream_contents.append(marshalled['data'])
2523 self.stream_file[k] = marshalled[k]
2526 'streamContentSize' in self.stream_file and
2527 'fileSize' in self.stream_file and
2528 'depotFile' in self.stream_file):
2529 size = int(self.stream_file["fileSize"])
2531 progress = 100*self.stream_file['streamContentSize']/size
2532 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2535 self.stream_have_file_info = True
2537 # Stream directly from "p4 files" into "git fast-import"
2538 def streamP4Files(self, files):
2544 # if using a client spec, only add the files that have
2545 # a path in the client
2546 if self.clientSpecDirs:
2547 if self.clientSpecDirs.map_in_client(f['path']) == "":
2550 filesForCommit.append(f)
2551 if f['action'] in self.delete_actions:
2552 filesToDelete.append(f)
2554 filesToRead.append(f)
2557 for f in filesToDelete:
2558 self.streamOneP4Deletion(f)
2560 if len(filesToRead) > 0:
2561 self.stream_file = {}
2562 self.stream_contents = []
2563 self.stream_have_file_info = False
2565 # curry self argument
2566 def streamP4FilesCbSelf(entry):
2567 self.streamP4FilesCb(entry)
2569 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2571 p4CmdList(["-x", "-", "print"],
2573 cb=streamP4FilesCbSelf)
2576 if self.stream_file.has_key('depotFile'):
2577 self.streamOneP4File(self.stream_file, self.stream_contents)
2579 def make_email(self, userid):
2580 if userid in self.users:
2581 return self.users[userid]
2583 return "%s <a@b>" % userid
2585 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2586 """ Stream a p4 tag.
2587 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2591 print "writing tag %s for commit %s" % (labelName, commit)
2592 gitStream.write("tag %s\n" % labelName)
2593 gitStream.write("from %s\n" % commit)
2595 if labelDetails.has_key('Owner'):
2596 owner = labelDetails["Owner"]
2600 # Try to use the owner of the p4 label, or failing that,
2601 # the current p4 user id.
2603 email = self.make_email(owner)
2605 email = self.make_email(self.p4UserId())
2606 tagger = "%s %s %s" % (email, epoch, self.tz)
2608 gitStream.write("tagger %s\n" % tagger)
2610 print "labelDetails=",labelDetails
2611 if labelDetails.has_key('Description'):
2612 description = labelDetails['Description']
2614 description = 'Label from git p4'
2616 gitStream.write("data %d\n" % len(description))
2617 gitStream.write(description)
2618 gitStream.write("\n")
2620 def commit(self, details, files, branch, parent = ""):
2621 epoch = details["time"]
2622 author = details["user"]
2625 print "commit into %s" % branch
2627 # start with reading files; if that fails, we should not
2631 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2632 new_files.append (f)
2634 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2636 if self.clientSpecDirs:
2637 self.clientSpecDirs.update_client_spec_path_cache(files)
2639 self.gitStream.write("commit %s\n" % branch)
2640 self.gitStream.write("mark :%s\n" % details["change"])
2641 self.committedChanges.add(int(details["change"]))
2643 if author not in self.users:
2644 self.getUserMapFromPerforceServer()
2645 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2647 self.gitStream.write("committer %s\n" % committer)
2649 self.gitStream.write("data <<EOT\n")
2650 self.gitStream.write(details["desc"])
2651 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2652 (','.join(self.branchPrefixes), details["change"]))
2653 if len(details['options']) > 0:
2654 self.gitStream.write(": options = %s" % details['options'])
2655 self.gitStream.write("]\nEOT\n\n")
2659 print "parent %s" % parent
2660 self.gitStream.write("from %s\n" % parent)
2662 self.streamP4Files(new_files)
2663 self.gitStream.write("\n")
2665 change = int(details["change"])
2667 if self.labels.has_key(change):
2668 label = self.labels[change]
2669 labelDetails = label[0]
2670 labelRevisions = label[1]
2672 print "Change %s is labelled %s" % (change, labelDetails)
2674 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2675 for p in self.branchPrefixes])
2677 if len(files) == len(labelRevisions):
2681 if info["action"] in self.delete_actions:
2683 cleanedFiles[info["depotFile"]] = info["rev"]
2685 if cleanedFiles == labelRevisions:
2686 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2690 print ("Tag %s does not match with change %s: files do not match."
2691 % (labelDetails["label"], change))
2695 print ("Tag %s does not match with change %s: file count is different."
2696 % (labelDetails["label"], change))
2698 # Build a dictionary of changelists and labels, for "detect-labels" option.
2699 def getLabels(self):
2702 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2703 if len(l) > 0 and not self.silent:
2704 print "Finding files belonging to labels in %s" % `self.depotPaths`
2707 label = output["label"]
2711 print "Querying files for label %s" % label
2712 for file in p4CmdList(["files"] +
2713 ["%s...@%s" % (p, label)
2714 for p in self.depotPaths]):
2715 revisions[file["depotFile"]] = file["rev"]
2716 change = int(file["change"])
2717 if change > newestChange:
2718 newestChange = change
2720 self.labels[newestChange] = [output, revisions]
2723 print "Label changes: %s" % self.labels.keys()
2725 # Import p4 labels as git tags. A direct mapping does not
2726 # exist, so assume that if all the files are at the same revision
2727 # then we can use that, or it's something more complicated we should
2729 def importP4Labels(self, stream, p4Labels):
2731 print "import p4 labels: " + ' '.join(p4Labels)
2733 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2734 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2735 if len(validLabelRegexp) == 0:
2736 validLabelRegexp = defaultLabelRegexp
2737 m = re.compile(validLabelRegexp)
2739 for name in p4Labels:
2742 if not m.match(name):
2744 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2747 if name in ignoredP4Labels:
2750 labelDetails = p4CmdList(['label', "-o", name])[0]
2752 # get the most recent changelist for each file in this label
2753 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2754 for p in self.depotPaths])
2756 if change.has_key('change'):
2757 # find the corresponding git commit; take the oldest commit
2758 changelist = int(change['change'])
2759 if changelist in self.committedChanges:
2760 gitCommit = ":%d" % changelist # use a fast-import mark
2763 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2764 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2765 if len(gitCommit) == 0:
2766 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2769 gitCommit = gitCommit.strip()
2772 # Convert from p4 time format
2774 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2776 print "Could not convert label time %s" % labelDetails['Update']
2779 when = int(time.mktime(tmwhen))
2780 self.streamTag(stream, name, labelDetails, gitCommit, when)
2782 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2785 print "Label %s has no changelists - possibly deleted?" % name
2788 # We can't import this label; don't try again as it will get very
2789 # expensive repeatedly fetching all the files for labels that will
2790 # never be imported. If the label is moved in the future, the
2791 # ignore will need to be removed manually.
2792 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2794 def guessProjectName(self):
2795 for p in self.depotPaths:
2798 p = p[p.strip().rfind("/") + 1:]
2799 if not p.endswith("/"):
2803 def getBranchMapping(self):
2804 lostAndFoundBranches = set()
2806 user = gitConfig("git-p4.branchUser")
2808 command = "branches -u %s" % user
2810 command = "branches"
2812 for info in p4CmdList(command):
2813 details = p4Cmd(["branch", "-o", info["branch"]])
2815 while details.has_key("View%s" % viewIdx):
2816 paths = details["View%s" % viewIdx].split(" ")
2817 viewIdx = viewIdx + 1
2818 # require standard //depot/foo/... //depot/bar/... mapping
2819 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2822 destination = paths[1]
2824 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2825 source = source[len(self.depotPaths[0]):-4]
2826 destination = destination[len(self.depotPaths[0]):-4]
2828 if destination in self.knownBranches:
2830 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2831 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2834 self.knownBranches[destination] = source
2836 lostAndFoundBranches.discard(destination)
2838 if source not in self.knownBranches:
2839 lostAndFoundBranches.add(source)
2841 # Perforce does not strictly require branches to be defined, so we also
2842 # check git config for a branch list.
2844 # Example of branch definition in git config file:
2846 # branchList=main:branchA
2847 # branchList=main:branchB
2848 # branchList=branchA:branchC
2849 configBranches = gitConfigList("git-p4.branchList")
2850 for branch in configBranches:
2852 (source, destination) = branch.split(":")
2853 self.knownBranches[destination] = source
2855 lostAndFoundBranches.discard(destination)
2857 if source not in self.knownBranches:
2858 lostAndFoundBranches.add(source)
2861 for branch in lostAndFoundBranches:
2862 self.knownBranches[branch] = branch
2864 def getBranchMappingFromGitBranches(self):
2865 branches = p4BranchesInGit(self.importIntoRemotes)
2866 for branch in branches.keys():
2867 if branch == "master":
2870 branch = branch[len(self.projectName):]
2871 self.knownBranches[branch] = branch
2873 def updateOptionDict(self, d):
2875 if self.keepRepoPath:
2876 option_keys['keepRepoPath'] = 1
2878 d["options"] = ' '.join(sorted(option_keys.keys()))
2880 def readOptions(self, d):
2881 self.keepRepoPath = (d.has_key('options')
2882 and ('keepRepoPath' in d['options']))
2884 def gitRefForBranch(self, branch):
2885 if branch == "main":
2886 return self.refPrefix + "master"
2888 if len(branch) <= 0:
2891 return self.refPrefix + self.projectName + branch
2893 def gitCommitByP4Change(self, ref, change):
2895 print "looking in ref " + ref + " for change %s using bisect..." % change
2898 latestCommit = parseRevision(ref)
2902 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2903 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2908 log = extractLogMessageFromGitCommit(next)
2909 settings = extractSettingsGitLog(log)
2910 currentChange = int(settings['change'])
2912 print "current change %s" % currentChange
2914 if currentChange == change:
2916 print "found %s" % next
2919 if currentChange < change:
2920 earliestCommit = "^%s" % next
2922 latestCommit = "%s" % next
2926 def importNewBranch(self, branch, maxChange):
2927 # make fast-import flush all changes to disk and update the refs using the checkpoint
2928 # command so that we can try to find the branch parent in the git history
2929 self.gitStream.write("checkpoint\n\n");
2930 self.gitStream.flush();
2931 branchPrefix = self.depotPaths[0] + branch + "/"
2932 range = "@1,%s" % maxChange
2933 #print "prefix" + branchPrefix
2934 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2935 if len(changes) <= 0:
2937 firstChange = changes[0]
2938 #print "first change in branch: %s" % firstChange
2939 sourceBranch = self.knownBranches[branch]
2940 sourceDepotPath = self.depotPaths[0] + sourceBranch
2941 sourceRef = self.gitRefForBranch(sourceBranch)
2942 #print "source " + sourceBranch
2944 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2945 #print "branch parent: %s" % branchParentChange
2946 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2947 if len(gitParent) > 0:
2948 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2949 #print "parent git commit: %s" % gitParent
2951 self.importChanges(changes)
2954 def searchParent(self, parent, branch, target):
2956 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2957 "--no-merges", parent]):
2959 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2962 print "Found parent of %s in commit %s" % (branch, blob)
2969 def importChanges(self, changes):
2971 for change in changes:
2972 description = p4_describe(change)
2973 self.updateOptionDict(description)
2976 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2981 if self.detectBranches:
2982 branches = self.splitFilesIntoBranches(description)
2983 for branch in branches.keys():
2985 branchPrefix = self.depotPaths[0] + branch + "/"
2986 self.branchPrefixes = [ branchPrefix ]
2990 filesForCommit = branches[branch]
2993 print "branch is %s" % branch
2995 self.updatedBranches.add(branch)
2997 if branch not in self.createdBranches:
2998 self.createdBranches.add(branch)
2999 parent = self.knownBranches[branch]
3000 if parent == branch:
3003 fullBranch = self.projectName + branch
3004 if fullBranch not in self.p4BranchesInGit:
3006 print("\n Importing new branch %s" % fullBranch);
3007 if self.importNewBranch(branch, change - 1):
3009 self.p4BranchesInGit.append(fullBranch)
3011 print("\n Resuming with change %s" % change);
3014 print "parent determined through known branches: %s" % parent
3016 branch = self.gitRefForBranch(branch)
3017 parent = self.gitRefForBranch(parent)
3020 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3022 if len(parent) == 0 and branch in self.initialParents:
3023 parent = self.initialParents[branch]
3024 del self.initialParents[branch]
3028 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3030 print "Creating temporary branch: " + tempBranch
3031 self.commit(description, filesForCommit, tempBranch)
3032 self.tempBranches.append(tempBranch)
3034 blob = self.searchParent(parent, branch, tempBranch)
3036 self.commit(description, filesForCommit, branch, blob)
3039 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3040 self.commit(description, filesForCommit, branch, parent)
3042 files = self.extractFilesFromCommit(description)
3043 self.commit(description, files, self.branch,
3045 # only needed once, to connect to the previous commit
3046 self.initialParent = ""
3048 print self.gitError.read()
3051 def importHeadRevision(self, revision):
3052 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3055 details["user"] = "git perforce import user"
3056 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3057 % (' '.join(self.depotPaths), revision))
3058 details["change"] = revision
3062 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3064 for info in p4CmdList(["files"] + fileArgs):
3066 if 'code' in info and info['code'] == 'error':
3067 sys.stderr.write("p4 returned an error: %s\n"
3069 if info['data'].find("must refer to client") >= 0:
3070 sys.stderr.write("This particular p4 error is misleading.\n")
3071 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3072 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3074 if 'p4ExitCode' in info:
3075 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3079 change = int(info["change"])
3080 if change > newestRevision:
3081 newestRevision = change
3083 if info["action"] in self.delete_actions:
3084 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3085 #fileCnt = fileCnt + 1
3088 for prop in ["depotFile", "rev", "action", "type" ]:
3089 details["%s%s" % (prop, fileCnt)] = info[prop]
3091 fileCnt = fileCnt + 1
3093 details["change"] = newestRevision
3095 # Use time from top-most change so that all git p4 clones of
3096 # the same p4 repo have the same commit SHA1s.
3097 res = p4_describe(newestRevision)
3098 details["time"] = res["time"]
3100 self.updateOptionDict(details)
3102 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3104 print "IO error with git fast-import. Is your git version recent enough?"
3105 print self.gitError.read()
3108 def run(self, args):
3109 self.depotPaths = []
3110 self.changeRange = ""
3111 self.previousDepotPaths = []
3112 self.hasOrigin = False
3114 # map from branch depot path to parent branch
3115 self.knownBranches = {}
3116 self.initialParents = {}
3118 if self.importIntoRemotes:
3119 self.refPrefix = "refs/remotes/p4/"
3121 self.refPrefix = "refs/heads/p4/"
3123 if self.syncWithOrigin:
3124 self.hasOrigin = originP4BranchesExist()
3127 print 'Syncing with origin first, using "git fetch origin"'
3128 system("git fetch origin")
3130 branch_arg_given = bool(self.branch)
3131 if len(self.branch) == 0:
3132 self.branch = self.refPrefix + "master"
3133 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3134 system("git update-ref %s refs/heads/p4" % self.branch)
3135 system("git branch -D p4")
3137 # accept either the command-line option, or the configuration variable
3138 if self.useClientSpec:
3139 # will use this after clone to set the variable
3140 self.useClientSpec_from_options = True
3142 if gitConfigBool("git-p4.useclientspec"):
3143 self.useClientSpec = True
3144 if self.useClientSpec:
3145 self.clientSpecDirs = getClientSpec()
3147 # TODO: should always look at previous commits,
3148 # merge with previous imports, if possible.
3151 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3153 # branches holds mapping from branch name to sha1
3154 branches = p4BranchesInGit(self.importIntoRemotes)
3156 # restrict to just this one, disabling detect-branches
3157 if branch_arg_given:
3158 short = self.branch.split("/")[-1]
3159 if short in branches:
3160 self.p4BranchesInGit = [ short ]
3162 self.p4BranchesInGit = branches.keys()
3164 if len(self.p4BranchesInGit) > 1:
3166 print "Importing from/into multiple branches"
3167 self.detectBranches = True
3168 for branch in branches.keys():
3169 self.initialParents[self.refPrefix + branch] = \
3173 print "branches: %s" % self.p4BranchesInGit
3176 for branch in self.p4BranchesInGit:
3177 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3179 settings = extractSettingsGitLog(logMsg)
3181 self.readOptions(settings)
3182 if (settings.has_key('depot-paths')
3183 and settings.has_key ('change')):
3184 change = int(settings['change']) + 1
3185 p4Change = max(p4Change, change)
3187 depotPaths = sorted(settings['depot-paths'])
3188 if self.previousDepotPaths == []:
3189 self.previousDepotPaths = depotPaths
3192 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3193 prev_list = prev.split("/")
3194 cur_list = cur.split("/")
3195 for i in range(0, min(len(cur_list), len(prev_list))):
3196 if cur_list[i] <> prev_list[i]:
3200 paths.append ("/".join(cur_list[:i + 1]))
3202 self.previousDepotPaths = paths
3205 self.depotPaths = sorted(self.previousDepotPaths)
3206 self.changeRange = "@%s,#head" % p4Change
3207 if not self.silent and not self.detectBranches:
3208 print "Performing incremental import into %s git branch" % self.branch
3210 # accept multiple ref name abbreviations:
3211 # refs/foo/bar/branch -> use it exactly
3212 # p4/branch -> prepend refs/remotes/ or refs/heads/
3213 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3214 if not self.branch.startswith("refs/"):
3215 if self.importIntoRemotes:
3216 prepend = "refs/remotes/"
3218 prepend = "refs/heads/"
3219 if not self.branch.startswith("p4/"):
3221 self.branch = prepend + self.branch
3223 if len(args) == 0 and self.depotPaths:
3225 print "Depot paths: %s" % ' '.join(self.depotPaths)
3227 if self.depotPaths and self.depotPaths != args:
3228 print ("previous import used depot path %s and now %s was specified. "
3229 "This doesn't work!" % (' '.join (self.depotPaths),
3233 self.depotPaths = sorted(args)
3238 # Make sure no revision specifiers are used when --changesfile
3240 bad_changesfile = False
3241 if len(self.changesFile) > 0:
3242 for p in self.depotPaths:
3243 if p.find("@") >= 0 or p.find("#") >= 0:
3244 bad_changesfile = True
3247 die("Option --changesfile is incompatible with revision specifiers")
3250 for p in self.depotPaths:
3251 if p.find("@") != -1:
3252 atIdx = p.index("@")
3253 self.changeRange = p[atIdx:]
3254 if self.changeRange == "@all":
3255 self.changeRange = ""
3256 elif ',' not in self.changeRange:
3257 revision = self.changeRange
3258 self.changeRange = ""
3260 elif p.find("#") != -1:
3261 hashIdx = p.index("#")
3262 revision = p[hashIdx:]
3264 elif self.previousDepotPaths == []:
3265 # pay attention to changesfile, if given, else import
3266 # the entire p4 tree at the head revision
3267 if len(self.changesFile) == 0:
3270 p = re.sub ("\.\.\.$", "", p)
3271 if not p.endswith("/"):
3276 self.depotPaths = newPaths
3278 # --detect-branches may change this for each branch
3279 self.branchPrefixes = self.depotPaths
3281 self.loadUserMapFromCache()
3283 if self.detectLabels:
3286 if self.detectBranches:
3287 ## FIXME - what's a P4 projectName ?
3288 self.projectName = self.guessProjectName()
3291 self.getBranchMappingFromGitBranches()
3293 self.getBranchMapping()
3295 print "p4-git branches: %s" % self.p4BranchesInGit
3296 print "initial parents: %s" % self.initialParents
3297 for b in self.p4BranchesInGit:
3301 b = b[len(self.projectName):]
3302 self.createdBranches.add(b)
3304 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3306 self.importProcess = subprocess.Popen(["git", "fast-import"],
3307 stdin=subprocess.PIPE,
3308 stdout=subprocess.PIPE,
3309 stderr=subprocess.PIPE);
3310 self.gitOutput = self.importProcess.stdout
3311 self.gitStream = self.importProcess.stdin
3312 self.gitError = self.importProcess.stderr
3315 self.importHeadRevision(revision)
3319 if len(self.changesFile) > 0:
3320 output = open(self.changesFile).readlines()
3323 changeSet.add(int(line))
3325 for change in changeSet:
3326 changes.append(change)
3330 # catch "git p4 sync" with no new branches, in a repo that
3331 # does not have any existing p4 branches
3333 if not self.p4BranchesInGit:
3334 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3336 # The default branch is master, unless --branch is used to
3337 # specify something else. Make sure it exists, or complain
3338 # nicely about how to use --branch.
3339 if not self.detectBranches:
3340 if not branch_exists(self.branch):
3341 if branch_arg_given:
3342 die("Error: branch %s does not exist." % self.branch)
3344 die("Error: no branch %s; perhaps specify one with --branch." %
3348 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3350 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3352 if len(self.maxChanges) > 0:
3353 changes = changes[:min(int(self.maxChanges), len(changes))]
3355 if len(changes) == 0:
3357 print "No changes to import!"
3359 if not self.silent and not self.detectBranches:
3360 print "Import destination: %s" % self.branch
3362 self.updatedBranches = set()
3364 if not self.detectBranches:
3366 # start a new branch
3367 self.initialParent = ""
3369 # build on a previous revision
3370 self.initialParent = parseRevision(self.branch)
3372 self.importChanges(changes)
3376 if len(self.updatedBranches) > 0:
3377 sys.stdout.write("Updated branches: ")
3378 for b in self.updatedBranches:
3379 sys.stdout.write("%s " % b)
3380 sys.stdout.write("\n")
3382 if gitConfigBool("git-p4.importLabels"):
3383 self.importLabels = True
3385 if self.importLabels:
3386 p4Labels = getP4Labels(self.depotPaths)
3387 gitTags = getGitTags()
3389 missingP4Labels = p4Labels - gitTags
3390 self.importP4Labels(self.gitStream, missingP4Labels)
3392 self.gitStream.close()
3393 if self.importProcess.wait() != 0:
3394 die("fast-import failed: %s" % self.gitError.read())
3395 self.gitOutput.close()
3396 self.gitError.close()
3398 # Cleanup temporary branches created during import
3399 if self.tempBranches != []:
3400 for branch in self.tempBranches:
3401 read_pipe("git update-ref -d %s" % branch)
3402 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3404 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3405 # a convenient shortcut refname "p4".
3406 if self.importIntoRemotes:
3407 head_ref = self.refPrefix + "HEAD"
3408 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3409 system(["git", "symbolic-ref", head_ref, self.branch])
3413 class P4Rebase(Command):
3415 Command.__init__(self)
3417 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3419 self.importLabels = False
3420 self.description = ("Fetches the latest revision from perforce and "
3421 + "rebases the current work (branch) against it")
3423 def run(self, args):
3425 sync.importLabels = self.importLabels
3428 return self.rebase()
3431 if os.system("git update-index --refresh") != 0:
3432 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3433 if len(read_pipe("git diff-index HEAD --")) > 0:
3434 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3436 [upstream, settings] = findUpstreamBranchPoint()
3437 if len(upstream) == 0:
3438 die("Cannot find upstream branchpoint for rebase")
3440 # the branchpoint may be p4/foo~3, so strip off the parent
3441 upstream = re.sub("~[0-9]+$", "", upstream)
3443 print "Rebasing the current branch onto %s" % upstream
3444 oldHead = read_pipe("git rev-parse HEAD").strip()
3445 system("git rebase %s" % upstream)
3446 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3449 class P4Clone(P4Sync):
3451 P4Sync.__init__(self)
3452 self.description = "Creates a new git repository and imports from Perforce into it"
3453 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3455 optparse.make_option("--destination", dest="cloneDestination",
3456 action='store', default=None,
3457 help="where to leave result of the clone"),
3458 optparse.make_option("--bare", dest="cloneBare",
3459 action="store_true", default=False),
3461 self.cloneDestination = None
3462 self.needsGit = False
3463 self.cloneBare = False
3465 def defaultDestination(self, args):
3466 ## TODO: use common prefix of args?
3468 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3469 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3470 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3471 depotDir = re.sub(r"/$", "", depotDir)
3472 return os.path.split(depotDir)[1]
3474 def run(self, args):
3478 if self.keepRepoPath and not self.cloneDestination:
3479 sys.stderr.write("Must specify destination for --keep-path\n")
3484 if not self.cloneDestination and len(depotPaths) > 1:
3485 self.cloneDestination = depotPaths[-1]
3486 depotPaths = depotPaths[:-1]
3488 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3489 for p in depotPaths:
3490 if not p.startswith("//"):
3491 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3494 if not self.cloneDestination:
3495 self.cloneDestination = self.defaultDestination(args)
3497 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3499 if not os.path.exists(self.cloneDestination):
3500 os.makedirs(self.cloneDestination)
3501 chdir(self.cloneDestination)
3503 init_cmd = [ "git", "init" ]
3505 init_cmd.append("--bare")
3506 retcode = subprocess.call(init_cmd)
3508 raise CalledProcessError(retcode, init_cmd)
3510 if not P4Sync.run(self, depotPaths):
3513 # create a master branch and check out a work tree
3514 if gitBranchExists(self.branch):
3515 system([ "git", "branch", "master", self.branch ])
3516 if not self.cloneBare:
3517 system([ "git", "checkout", "-f" ])
3519 print 'Not checking out any branch, use ' \
3520 '"git checkout -q -b master <branch>"'
3522 # auto-set this variable if invoked with --use-client-spec
3523 if self.useClientSpec_from_options:
3524 system("git config --bool git-p4.useclientspec true")
3528 class P4Branches(Command):
3530 Command.__init__(self)
3532 self.description = ("Shows the git branches that hold imports and their "
3533 + "corresponding perforce depot paths")
3534 self.verbose = False
3536 def run(self, args):
3537 if originP4BranchesExist():
3538 createOrUpdateBranchesFromOrigin()
3540 cmdline = "git rev-parse --symbolic "
3541 cmdline += " --remotes"
3543 for line in read_pipe_lines(cmdline):
3546 if not line.startswith('p4/') or line == "p4/HEAD":
3550 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3551 settings = extractSettingsGitLog(log)
3553 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3556 class HelpFormatter(optparse.IndentedHelpFormatter):
3558 optparse.IndentedHelpFormatter.__init__(self)
3560 def format_description(self, description):
3562 return description + "\n"
3566 def printUsage(commands):
3567 print "usage: %s <command> [options]" % sys.argv[0]
3569 print "valid commands: %s" % ", ".join(commands)
3571 print "Try %s <command> --help for command specific help." % sys.argv[0]
3576 "submit" : P4Submit,
3577 "commit" : P4Submit,
3579 "rebase" : P4Rebase,
3581 "rollback" : P4RollBack,
3582 "branches" : P4Branches
3587 if len(sys.argv[1:]) == 0:
3588 printUsage(commands.keys())
3591 cmdName = sys.argv[1]
3593 klass = commands[cmdName]
3596 print "unknown command %s" % cmdName
3598 printUsage(commands.keys())
3601 options = cmd.options
3602 cmd.gitdir = os.environ.get("GIT_DIR", None)
3606 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3608 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3610 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3612 description = cmd.description,
3613 formatter = HelpFormatter())
3615 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3617 verbose = cmd.verbose
3619 if cmd.gitdir == None:
3620 cmd.gitdir = os.path.abspath(".git")
3621 if not isValidGitDir(cmd.gitdir):
3622 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3623 if os.path.exists(cmd.gitdir):
3624 cdup = read_pipe("git rev-parse --show-cdup").strip()
3628 if not isValidGitDir(cmd.gitdir):
3629 if isValidGitDir(cmd.gitdir + "/.git"):
3630 cmd.gitdir += "/.git"
3632 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3634 os.environ["GIT_DIR"] = cmd.gitdir
3636 if not cmd.run(args):
3641 if __name__ == '__main__':