3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
31 from subprocess import CalledProcessError
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
62 user = gitConfig("git-p4.user")
64 real_cmd += ["-u",user]
66 password = gitConfig("git-p4.password")
68 real_cmd += ["-P", password]
70 port = gitConfig("git-p4.port")
72 real_cmd += ["-p", port]
74 host = gitConfig("git-p4.host")
76 real_cmd += ["-H", host]
78 client = gitConfig("git-p4.client")
80 real_cmd += ["-c", client]
82 retries = gitConfigInt("git-p4.retries")
84 # Perform 3 retries by default
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
106 def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
124 if not is_client_path:
126 os.environ['PWD'] = path
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
142 sys.stderr.write(msg + "\n")
145 def write_pipe(c, stdin):
147 sys.stderr.write('Writing pipe: %s\n' % str(c))
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
152 val = pipe.write(stdin)
155 die('Command failed: %s' % str(c))
159 def p4_write_pipe(c, stdin):
160 real_cmd = p4_build_cmd(c)
161 return write_pipe(real_cmd, stdin)
163 def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
169 sys.stderr.write('Reading pipe: %s\n' % str(c))
171 expand = isinstance(c,basestring)
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
174 return (p.returncode, out, err)
176 def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
181 (retcode, out, err) = read_pipe_full(c)
186 die('Command failed: %s\nError: %s' % (str(c), err))
189 def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
193 (retcode, out, err) = read_pipe_full(c)
199 def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
203 def read_pipe_lines(c):
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
210 val = pipe.readlines()
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
216 def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
218 real_cmd = p4_build_cmd(c)
219 return read_pipe_lines(real_cmd)
221 def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
228 return p.returncode == 0
230 def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
236 if not p4_has_command("move"):
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
244 if err.find("disabled") >= 0:
246 # assume it failed because @... was invalid changelist
249 def system(cmd, ignore_error=False):
250 expand = isinstance(cmd,basestring)
252 sys.stderr.write("executing %s\n" % str(cmd))
253 retcode = subprocess.call(cmd, shell=expand)
254 if retcode and not ignore_error:
255 raise CalledProcessError(retcode, cmd)
260 """Specifically invoke p4 as the system command. """
261 real_cmd = p4_build_cmd(cmd)
262 expand = isinstance(real_cmd, basestring)
263 retcode = subprocess.call(real_cmd, shell=expand)
265 raise CalledProcessError(retcode, real_cmd)
267 _p4_version_string = None
268 def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
283 def p4_integrate(src, dest):
284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
286 def p4_sync(f, *options):
287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
294 p4_system(["add", f])
297 p4_system(["delete", wildcard_encode(f)])
299 def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
303 p4_system(["revert", wildcard_encode(f)])
305 def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
308 def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
312 def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
315 def p4_last_change():
316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317 return int(results[0]['change'])
319 def p4_describe(change, shelved=False):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
324 cmd = ["describe", "-s"]
329 ds = p4CmdList(cmd, skip_info=True)
331 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
335 if "p4ExitCode" in d:
336 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
339 if d["code"] == "error":
340 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
343 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
348 # Canonicalize the p4 type and return a tuple of the
349 # base type, plus any modifiers. See "p4 help filetypes"
350 # for a list and explanation.
352 def split_p4_type(p4type):
354 p4_filetypes_historical = {
355 "ctempobj": "binary+Sw",
361 "tempobj": "binary+FSw",
362 "ubinary": "binary+F",
363 "uresource": "resource+F",
364 "uxbinary": "binary+Fx",
365 "xbinary": "binary+x",
367 "xtempobj": "binary+Swx",
369 "xunicode": "unicode+x",
372 if p4type in p4_filetypes_historical:
373 p4type = p4_filetypes_historical[p4type]
375 s = p4type.split("+")
383 # return the raw p4 type of a file (text, text+ko, etc)
386 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
387 return results[0]['headType']
390 # Given a type base and modifier, return a regexp matching
391 # the keywords that can be expanded in the file
393 def p4_keywords_regexp_for_type(base, type_mods):
394 if base in ("text", "unicode", "binary"):
396 if "ko" in type_mods:
398 elif "k" in type_mods:
399 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
403 \$ # Starts with a dollar, followed by...
404 (%s) # one of the keywords, followed by...
405 (:[^$\n]+)? # possibly an old expansion, followed by...
413 # Given a file, return a regexp matching the possible
414 # RCS keywords that will be expanded, or None for files
415 # with kw expansion turned off.
417 def p4_keywords_regexp_for_file(file):
418 if not os.path.exists(file):
421 (type_base, type_mods) = split_p4_type(p4_type(file))
422 return p4_keywords_regexp_for_type(type_base, type_mods)
424 def setP4ExecBit(file, mode):
425 # Reopens an already open file and changes the execute bit to match
426 # the execute bit setting in the passed in mode.
430 if not isModeExec(mode):
431 p4Type = getP4OpenedType(file)
432 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
433 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
434 if p4Type[-1] == "+":
435 p4Type = p4Type[0:-1]
437 p4_reopen(p4Type, file)
439 def getP4OpenedType(file):
440 # Returns the perforce file type for the given file.
442 result = p4_read_pipe(["opened", wildcard_encode(file)])
443 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
445 return match.group(1)
447 die("Could not determine file type for %s (result: '%s')" % (file, result))
449 # Return the set of all p4 labels
450 def getP4Labels(depotPaths):
452 if isinstance(depotPaths,basestring):
453 depotPaths = [depotPaths]
455 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
461 # Return the set of all git tags
464 for line in read_pipe_lines(["git", "tag"]):
469 def diffTreePattern():
470 # This is a simple generator for the diff tree regex pattern. This could be
471 # a class variable if this and parseDiffTreeEntry were a part of a class.
472 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
476 def parseDiffTreeEntry(entry):
477 """Parses a single diff tree entry into its component elements.
479 See git-diff-tree(1) manpage for details about the format of the diff
480 output. This method returns a dictionary with the following elements:
482 src_mode - The mode of the source file
483 dst_mode - The mode of the destination file
484 src_sha1 - The sha1 for the source file
485 dst_sha1 - The sha1 fr the destination file
486 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
487 status_score - The score for the status (applicable for 'C' and 'R'
488 statuses). This is None if there is no score.
489 src - The path for the source file.
490 dst - The path for the destination file. This is only present for
491 copy or renames. If it is not present, this is None.
493 If the pattern is not matched, None is returned."""
495 match = diffTreePattern().next().match(entry)
498 'src_mode': match.group(1),
499 'dst_mode': match.group(2),
500 'src_sha1': match.group(3),
501 'dst_sha1': match.group(4),
502 'status': match.group(5),
503 'status_score': match.group(6),
504 'src': match.group(7),
505 'dst': match.group(10)
509 def isModeExec(mode):
510 # Returns True if the given git mode represents an executable file,
512 return mode[-3:] == "755"
514 def isModeExecChanged(src_mode, dst_mode):
515 return isModeExec(src_mode) != isModeExec(dst_mode)
517 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
519 if isinstance(cmd,basestring):
526 cmd = p4_build_cmd(cmd)
528 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
530 # Use a temporary file to avoid deadlocks without
531 # subprocess.communicate(), which would put another copy
532 # of stdout into memory.
534 if stdin is not None:
535 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
536 if isinstance(stdin,basestring):
537 stdin_file.write(stdin)
540 stdin_file.write(i + '\n')
544 p4 = subprocess.Popen(cmd,
547 stdout=subprocess.PIPE)
552 entry = marshal.load(p4.stdout)
554 if 'code' in entry and entry['code'] == 'info':
565 entry["p4ExitCode"] = exitCode
571 list = p4CmdList(cmd)
577 def p4Where(depotPath):
578 if not depotPath.endswith("/"):
580 depotPathLong = depotPath + "..."
581 outputList = p4CmdList(["where", depotPathLong])
583 for entry in outputList:
584 if "depotFile" in entry:
585 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
586 # The base path always ends with "/...".
587 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
590 elif "data" in entry:
591 data = entry.get("data")
592 space = data.find(" ")
593 if data[:space] == depotPath:
598 if output["code"] == "error":
602 clientPath = output.get("path")
603 elif "data" in output:
604 data = output.get("data")
605 lastSpace = data.rfind(" ")
606 clientPath = data[lastSpace + 1:]
608 if clientPath.endswith("..."):
609 clientPath = clientPath[:-3]
612 def currentGitBranch():
613 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
615 def isValidGitDir(path):
616 return git_dir(path) != None
618 def parseRevision(ref):
619 return read_pipe("git rev-parse %s" % ref).strip()
621 def branchExists(ref):
622 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
626 def extractLogMessageFromGitCommit(commit):
629 ## fixme: title is first line of commit, not 1st paragraph.
631 for log in read_pipe_lines("git cat-file commit %s" % commit):
640 def extractSettingsGitLog(log):
642 for line in log.split("\n"):
644 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
648 assignments = m.group(1).split (':')
649 for a in assignments:
651 key = vals[0].strip()
652 val = ('='.join (vals[1:])).strip()
653 if val.endswith ('\"') and val.startswith('"'):
658 paths = values.get("depot-paths")
660 paths = values.get("depot-path")
662 values['depot-paths'] = paths.split(',')
665 def gitBranchExists(branch):
666 proc = subprocess.Popen(["git", "rev-parse", branch],
667 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
668 return proc.wait() == 0;
672 def gitConfig(key, typeSpecifier=None):
673 if not _gitConfig.has_key(key):
674 cmd = [ "git", "config" ]
676 cmd += [ typeSpecifier ]
678 s = read_pipe(cmd, ignore_error=True)
679 _gitConfig[key] = s.strip()
680 return _gitConfig[key]
682 def gitConfigBool(key):
683 """Return a bool, using git config --bool. It is True only if the
684 variable is set to true, and False if set to false or not present
687 if not _gitConfig.has_key(key):
688 _gitConfig[key] = gitConfig(key, '--bool') == "true"
689 return _gitConfig[key]
691 def gitConfigInt(key):
692 if not _gitConfig.has_key(key):
693 cmd = [ "git", "config", "--int", key ]
694 s = read_pipe(cmd, ignore_error=True)
697 _gitConfig[key] = int(gitConfig(key, '--int'))
699 _gitConfig[key] = None
700 return _gitConfig[key]
702 def gitConfigList(key):
703 if not _gitConfig.has_key(key):
704 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
705 _gitConfig[key] = s.strip().splitlines()
706 if _gitConfig[key] == ['']:
708 return _gitConfig[key]
710 def p4BranchesInGit(branchesAreInRemotes=True):
711 """Find all the branches whose names start with "p4/", looking
712 in remotes or heads as specified by the argument. Return
713 a dictionary of { branch: revision } for each one found.
714 The branch names are the short names, without any
719 cmdline = "git rev-parse --symbolic "
720 if branchesAreInRemotes:
721 cmdline += "--remotes"
723 cmdline += "--branches"
725 for line in read_pipe_lines(cmdline):
729 if not line.startswith('p4/'):
731 # special symbolic ref to p4/master
732 if line == "p4/HEAD":
735 # strip off p4/ prefix
736 branch = line[len("p4/"):]
738 branches[branch] = parseRevision(line)
742 def branch_exists(branch):
743 """Make sure that the given ref name really exists."""
745 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
746 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
747 out, _ = p.communicate()
750 # expect exactly one line of output: the branch name
751 return out.rstrip() == branch
753 def findUpstreamBranchPoint(head = "HEAD"):
754 branches = p4BranchesInGit()
755 # map from depot-path to branch name
756 branchByDepotPath = {}
757 for branch in branches.keys():
758 tip = branches[branch]
759 log = extractLogMessageFromGitCommit(tip)
760 settings = extractSettingsGitLog(log)
761 if settings.has_key("depot-paths"):
762 paths = ",".join(settings["depot-paths"])
763 branchByDepotPath[paths] = "remotes/p4/" + branch
767 while parent < 65535:
768 commit = head + "~%s" % parent
769 log = extractLogMessageFromGitCommit(commit)
770 settings = extractSettingsGitLog(log)
771 if settings.has_key("depot-paths"):
772 paths = ",".join(settings["depot-paths"])
773 if branchByDepotPath.has_key(paths):
774 return [branchByDepotPath[paths], settings]
778 return ["", settings]
780 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
782 print ("Creating/updating branch(es) in %s based on origin branch(es)"
785 originPrefix = "origin/p4/"
787 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
789 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
792 headName = line[len(originPrefix):]
793 remoteHead = localRefPrefix + headName
796 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
797 if (not original.has_key('depot-paths')
798 or not original.has_key('change')):
802 if not gitBranchExists(remoteHead):
804 print "creating %s" % remoteHead
807 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
808 if settings.has_key('change') > 0:
809 if settings['depot-paths'] == original['depot-paths']:
810 originP4Change = int(original['change'])
811 p4Change = int(settings['change'])
812 if originP4Change > p4Change:
813 print ("%s (%s) is newer than %s (%s). "
814 "Updating p4 branch from origin."
815 % (originHead, originP4Change,
816 remoteHead, p4Change))
819 print ("Ignoring: %s was imported from %s while "
820 "%s was imported from %s"
821 % (originHead, ','.join(original['depot-paths']),
822 remoteHead, ','.join(settings['depot-paths'])))
825 system("git update-ref %s %s" % (remoteHead, originHead))
827 def originP4BranchesExist():
828 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
831 def p4ParseNumericChangeRange(parts):
832 changeStart = int(parts[0][1:])
833 if parts[1] == '#head':
834 changeEnd = p4_last_change()
836 changeEnd = int(parts[1])
838 return (changeStart, changeEnd)
840 def chooseBlockSize(blockSize):
844 return defaultBlockSize
846 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
849 # Parse the change range into start and end. Try to find integer
850 # revision ranges as these can be broken up into blocks to avoid
851 # hitting server-side limits (maxrows, maxscanresults). But if
852 # that doesn't work, fall back to using the raw revision specifier
853 # strings, without using block mode.
855 if changeRange is None or changeRange == '':
857 changeEnd = p4_last_change()
858 block_size = chooseBlockSize(requestedBlockSize)
860 parts = changeRange.split(',')
861 assert len(parts) == 2
863 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
864 block_size = chooseBlockSize(requestedBlockSize)
866 changeStart = parts[0][1:]
868 if requestedBlockSize:
869 die("cannot use --changes-block-size with non-numeric revisions")
874 # Retrieve changes a block at a time, to prevent running
875 # into a MaxResults/MaxScanRows error from the server.
881 end = min(changeEnd, changeStart + block_size)
882 revisionRange = "%d,%d" % (changeStart, end)
884 revisionRange = "%s,%s" % (changeStart, changeEnd)
887 cmd += ["%s...@%s" % (p, revisionRange)]
889 # Insert changes in chronological order
890 for entry in reversed(p4CmdList(cmd)):
891 if entry.has_key('p4ExitCode'):
892 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
893 if not entry.has_key('change'):
895 changes.add(int(entry['change']))
903 changeStart = end + 1
905 changes = sorted(changes)
908 def p4PathStartsWith(path, prefix):
909 # This method tries to remedy a potential mixed-case issue:
911 # If UserA adds //depot/DirA/file1
912 # and UserB adds //depot/dira/file2
914 # we may or may not have a problem. If you have core.ignorecase=true,
915 # we treat DirA and dira as the same directory
916 if gitConfigBool("core.ignorecase"):
917 return path.lower().startswith(prefix.lower())
918 return path.startswith(prefix)
921 """Look at the p4 client spec, create a View() object that contains
922 all the mappings, and return it."""
924 specList = p4CmdList("client -o")
925 if len(specList) != 1:
926 die('Output from "client -o" is %d lines, expecting 1' %
929 # dictionary of all client parameters
933 client_name = entry["Client"]
935 # just the keys that start with "View"
936 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
939 view = View(client_name)
941 # append the lines, in order, to the view
942 for view_num in range(len(view_keys)):
943 k = "View%d" % view_num
944 if k not in view_keys:
945 die("Expected view key %s missing" % k)
946 view.append(entry[k])
951 """Grab the client directory."""
953 output = p4CmdList("client -o")
955 die('Output from "client -o" is %d lines, expecting 1' % len(output))
958 if "Root" not in entry:
959 die('Client has no "Root"')
964 # P4 wildcards are not allowed in filenames. P4 complains
965 # if you simply add them, but you can force it with "-f", in
966 # which case it translates them into %xx encoding internally.
968 def wildcard_decode(path):
969 # Search for and fix just these four characters. Do % last so
970 # that fixing it does not inadvertently create new %-escapes.
971 # Cannot have * in a filename in windows; untested as to
972 # what p4 would do in such a case.
973 if not platform.system() == "Windows":
974 path = path.replace("%2A", "*")
975 path = path.replace("%23", "#") \
976 .replace("%40", "@") \
980 def wildcard_encode(path):
981 # do % first to avoid double-encoding the %s introduced here
982 path = path.replace("%", "%25") \
983 .replace("*", "%2A") \
984 .replace("#", "%23") \
988 def wildcard_present(path):
989 m = re.search("[*#@%]", path)
992 class LargeFileSystem(object):
993 """Base class for large file system support."""
995 def __init__(self, writeToGitStream):
996 self.largeFiles = set()
997 self.writeToGitStream = writeToGitStream
999 def generatePointer(self, cloneDestination, contentFile):
1000 """Return the content of a pointer file that is stored in Git instead of
1001 the actual content."""
1002 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1004 def pushFile(self, localLargeFile):
1005 """Push the actual content which is not stored in the Git repository to
1007 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1009 def hasLargeFileExtension(self, relPath):
1011 lambda a, b: a or b,
1012 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1016 def generateTempFile(self, contents):
1017 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1019 contentFile.write(d)
1021 return contentFile.name
1023 def exceedsLargeFileThreshold(self, relPath, contents):
1024 if gitConfigInt('git-p4.largeFileThreshold'):
1025 contentsSize = sum(len(d) for d in contents)
1026 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1028 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1029 contentsSize = sum(len(d) for d in contents)
1030 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1032 contentTempFile = self.generateTempFile(contents)
1033 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1034 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1035 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1037 compressedContentsSize = zf.infolist()[0].compress_size
1038 os.remove(contentTempFile)
1039 os.remove(compressedContentFile.name)
1040 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1044 def addLargeFile(self, relPath):
1045 self.largeFiles.add(relPath)
1047 def removeLargeFile(self, relPath):
1048 self.largeFiles.remove(relPath)
1050 def isLargeFile(self, relPath):
1051 return relPath in self.largeFiles
1053 def processContent(self, git_mode, relPath, contents):
1054 """Processes the content of git fast import. This method decides if a
1055 file is stored in the large file system and handles all necessary
1057 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1058 contentTempFile = self.generateTempFile(contents)
1059 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1060 if pointer_git_mode:
1061 git_mode = pointer_git_mode
1063 # Move temp file to final location in large file system
1064 largeFileDir = os.path.dirname(localLargeFile)
1065 if not os.path.isdir(largeFileDir):
1066 os.makedirs(largeFileDir)
1067 shutil.move(contentTempFile, localLargeFile)
1068 self.addLargeFile(relPath)
1069 if gitConfigBool('git-p4.largeFilePush'):
1070 self.pushFile(localLargeFile)
1072 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1073 return (git_mode, contents)
1075 class MockLFS(LargeFileSystem):
1076 """Mock large file system for testing."""
1078 def generatePointer(self, contentFile):
1079 """The pointer content is the original content prefixed with "pointer-".
1080 The local filename of the large file storage is derived from the file content.
1082 with open(contentFile, 'r') as f:
1085 pointerContents = 'pointer-' + content
1086 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1087 return (gitMode, pointerContents, localLargeFile)
1089 def pushFile(self, localLargeFile):
1090 """The remote filename of the large file storage is the same as the local
1091 one but in a different directory.
1093 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1094 if not os.path.exists(remotePath):
1095 os.makedirs(remotePath)
1096 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1098 class GitLFS(LargeFileSystem):
1099 """Git LFS as backend for the git-p4 large file system.
1100 See https://git-lfs.github.com/ for details."""
1102 def __init__(self, *args):
1103 LargeFileSystem.__init__(self, *args)
1104 self.baseGitAttributes = []
1106 def generatePointer(self, contentFile):
1107 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1108 mode and content which is stored in the Git repository instead of
1109 the actual content. Return also the new location of the actual
1112 if os.path.getsize(contentFile) == 0:
1113 return (None, '', None)
1115 pointerProcess = subprocess.Popen(
1116 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1117 stdout=subprocess.PIPE
1119 pointerFile = pointerProcess.stdout.read()
1120 if pointerProcess.wait():
1121 os.remove(contentFile)
1122 die('git-lfs pointer command failed. Did you install the extension?')
1124 # Git LFS removed the preamble in the output of the 'pointer' command
1125 # starting from version 1.2.0. Check for the preamble here to support
1127 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1128 if pointerFile.startswith('Git LFS pointer for'):
1129 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1131 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1132 localLargeFile = os.path.join(
1134 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1137 # LFS Spec states that pointer files should not have the executable bit set.
1139 return (gitMode, pointerFile, localLargeFile)
1141 def pushFile(self, localLargeFile):
1142 uploadProcess = subprocess.Popen(
1143 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1145 if uploadProcess.wait():
1146 die('git-lfs push command failed. Did you define a remote?')
1148 def generateGitAttributes(self):
1150 self.baseGitAttributes +
1154 '# Git LFS (see https://git-lfs.github.com/)\n',
1157 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1158 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1160 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1161 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1165 def addLargeFile(self, relPath):
1166 LargeFileSystem.addLargeFile(self, relPath)
1167 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1169 def removeLargeFile(self, relPath):
1170 LargeFileSystem.removeLargeFile(self, relPath)
1171 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1173 def processContent(self, git_mode, relPath, contents):
1174 if relPath == '.gitattributes':
1175 self.baseGitAttributes = contents
1176 return (git_mode, self.generateGitAttributes())
1178 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1182 self.usage = "usage: %prog [options]"
1183 self.needsGit = True
1184 self.verbose = False
1186 # This is required for the "append" cloneExclude action
1187 def ensure_value(self, attr, value):
1188 if not hasattr(self, attr) or getattr(self, attr) is None:
1189 setattr(self, attr, value)
1190 return getattr(self, attr)
1194 self.userMapFromPerforceServer = False
1195 self.myP4UserId = None
1199 return self.myP4UserId
1201 results = p4CmdList("user -o")
1203 if r.has_key('User'):
1204 self.myP4UserId = r['User']
1206 die("Could not find your p4 user id")
1208 def p4UserIsMe(self, p4User):
1209 # return True if the given p4 user is actually me
1210 me = self.p4UserId()
1211 if not p4User or p4User != me:
1216 def getUserCacheFilename(self):
1217 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1218 return home + "/.gitp4-usercache.txt"
1220 def getUserMapFromPerforceServer(self):
1221 if self.userMapFromPerforceServer:
1226 for output in p4CmdList("users"):
1227 if not output.has_key("User"):
1229 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1230 self.emails[output["Email"]] = output["User"]
1232 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1233 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1234 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1235 if mapUser and len(mapUser[0]) == 3:
1236 user = mapUser[0][0]
1237 fullname = mapUser[0][1]
1238 email = mapUser[0][2]
1239 self.users[user] = fullname + " <" + email + ">"
1240 self.emails[email] = user
1243 for (key, val) in self.users.items():
1244 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1246 open(self.getUserCacheFilename(), "wb").write(s)
1247 self.userMapFromPerforceServer = True
1249 def loadUserMapFromCache(self):
1251 self.userMapFromPerforceServer = False
1253 cache = open(self.getUserCacheFilename(), "rb")
1254 lines = cache.readlines()
1257 entry = line.strip().split("\t")
1258 self.users[entry[0]] = entry[1]
1260 self.getUserMapFromPerforceServer()
1262 class P4Debug(Command):
1264 Command.__init__(self)
1266 self.description = "A tool to debug the output of p4 -G."
1267 self.needsGit = False
1269 def run(self, args):
1271 for output in p4CmdList(args):
1272 print 'Element: %d' % j
1277 class P4RollBack(Command):
1279 Command.__init__(self)
1281 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1283 self.description = "A tool to debug the multi-branch import. Don't use :)"
1284 self.rollbackLocalBranches = False
1286 def run(self, args):
1289 maxChange = int(args[0])
1291 if "p4ExitCode" in p4Cmd("changes -m 1"):
1292 die("Problems executing p4");
1294 if self.rollbackLocalBranches:
1295 refPrefix = "refs/heads/"
1296 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1298 refPrefix = "refs/remotes/"
1299 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1302 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1304 ref = refPrefix + line
1305 log = extractLogMessageFromGitCommit(ref)
1306 settings = extractSettingsGitLog(log)
1308 depotPaths = settings['depot-paths']
1309 change = settings['change']
1313 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1314 for p in depotPaths]))) == 0:
1315 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1316 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1319 while change and int(change) > maxChange:
1322 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1323 system("git update-ref %s \"%s^\"" % (ref, ref))
1324 log = extractLogMessageFromGitCommit(ref)
1325 settings = extractSettingsGitLog(log)
1328 depotPaths = settings['depot-paths']
1329 change = settings['change']
1332 print "%s rewound to %s" % (ref, change)
1336 class P4Submit(Command, P4UserMap):
1338 conflict_behavior_choices = ("ask", "skip", "quit")
1341 Command.__init__(self)
1342 P4UserMap.__init__(self)
1344 optparse.make_option("--origin", dest="origin"),
1345 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1346 # preserve the user, requires relevant p4 permissions
1347 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1348 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1349 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1350 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1351 optparse.make_option("--conflict", dest="conflict_behavior",
1352 choices=self.conflict_behavior_choices),
1353 optparse.make_option("--branch", dest="branch"),
1354 optparse.make_option("--shelve", dest="shelve", action="store_true",
1355 help="Shelve instead of submit. Shelved files are reverted, "
1356 "restoring the workspace to the state before the shelve"),
1357 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1358 metavar="CHANGELIST",
1359 help="update an existing shelved changelist, implies --shelve, "
1360 "repeat in-order for multiple shelved changelists")
1362 self.description = "Submit changes from git to the perforce depot."
1363 self.usage += " [name of git branch to submit into perforce depot]"
1365 self.detectRenames = False
1366 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1367 self.dry_run = False
1369 self.update_shelve = list()
1370 self.prepare_p4_only = False
1371 self.conflict_behavior = None
1372 self.isWindows = (platform.system() == "Windows")
1373 self.exportLabels = False
1374 self.p4HasMoveCommand = p4_has_move_command()
1377 if gitConfig('git-p4.largeFileSystem'):
1378 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1381 if len(p4CmdList("opened ...")) > 0:
1382 die("You have files opened with perforce! Close them before starting the sync.")
1384 def separate_jobs_from_description(self, message):
1385 """Extract and return a possible Jobs field in the commit
1386 message. It goes into a separate section in the p4 change
1389 A jobs line starts with "Jobs:" and looks like a new field
1390 in a form. Values are white-space separated on the same
1391 line or on following lines that start with a tab.
1393 This does not parse and extract the full git commit message
1394 like a p4 form. It just sees the Jobs: line as a marker
1395 to pass everything from then on directly into the p4 form,
1396 but outside the description section.
1398 Return a tuple (stripped log message, jobs string)."""
1400 m = re.search(r'^Jobs:', message, re.MULTILINE)
1402 return (message, None)
1404 jobtext = message[m.start():]
1405 stripped_message = message[:m.start()].rstrip()
1406 return (stripped_message, jobtext)
1408 def prepareLogMessage(self, template, message, jobs):
1409 """Edits the template returned from "p4 change -o" to insert
1410 the message in the Description field, and the jobs text in
1414 inDescriptionSection = False
1416 for line in template.split("\n"):
1417 if line.startswith("#"):
1418 result += line + "\n"
1421 if inDescriptionSection:
1422 if line.startswith("Files:") or line.startswith("Jobs:"):
1423 inDescriptionSection = False
1424 # insert Jobs section
1426 result += jobs + "\n"
1430 if line.startswith("Description:"):
1431 inDescriptionSection = True
1433 for messageLine in message.split("\n"):
1434 line += "\t" + messageLine + "\n"
1436 result += line + "\n"
1440 def patchRCSKeywords(self, file, pattern):
1441 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1442 (handle, outFileName) = tempfile.mkstemp(dir='.')
1444 outFile = os.fdopen(handle, "w+")
1445 inFile = open(file, "r")
1446 regexp = re.compile(pattern, re.VERBOSE)
1447 for line in inFile.readlines():
1448 line = regexp.sub(r'$\1$', line)
1452 # Forcibly overwrite the original file
1454 shutil.move(outFileName, file)
1456 # cleanup our temporary file
1457 os.unlink(outFileName)
1458 print "Failed to strip RCS keywords in %s" % file
1461 print "Patched up RCS keywords in %s" % file
1463 def p4UserForCommit(self,id):
1464 # Return the tuple (perforce user,git email) for a given git commit id
1465 self.getUserMapFromPerforceServer()
1466 gitEmail = read_pipe(["git", "log", "--max-count=1",
1467 "--format=%ae", id])
1468 gitEmail = gitEmail.strip()
1469 if not self.emails.has_key(gitEmail):
1470 return (None,gitEmail)
1472 return (self.emails[gitEmail],gitEmail)
1474 def checkValidP4Users(self,commits):
1475 # check if any git authors cannot be mapped to p4 users
1477 (user,email) = self.p4UserForCommit(id)
1479 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1480 if gitConfigBool("git-p4.allowMissingP4Users"):
1483 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1485 def lastP4Changelist(self):
1486 # Get back the last changelist number submitted in this client spec. This
1487 # then gets used to patch up the username in the change. If the same
1488 # client spec is being used by multiple processes then this might go
1490 results = p4CmdList("client -o") # find the current client
1493 if r.has_key('Client'):
1494 client = r['Client']
1497 die("could not get client spec")
1498 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1500 if r.has_key('change'):
1502 die("Could not get changelist number for last submit - cannot patch up user details")
1504 def modifyChangelistUser(self, changelist, newUser):
1505 # fixup the user field of a changelist after it has been submitted.
1506 changes = p4CmdList("change -o %s" % changelist)
1507 if len(changes) != 1:
1508 die("Bad output from p4 change modifying %s to user %s" %
1509 (changelist, newUser))
1512 if c['User'] == newUser: return # nothing to do
1514 input = marshal.dumps(c)
1516 result = p4CmdList("change -f -i", stdin=input)
1518 if r.has_key('code'):
1519 if r['code'] == 'error':
1520 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1521 if r.has_key('data'):
1522 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1524 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1526 def canChangeChangelists(self):
1527 # check to see if we have p4 admin or super-user permissions, either of
1528 # which are required to modify changelists.
1529 results = p4CmdList(["protects", self.depotPath])
1531 if r.has_key('perm'):
1532 if r['perm'] == 'admin':
1534 if r['perm'] == 'super':
1538 def prepareSubmitTemplate(self, changelist=None):
1539 """Run "p4 change -o" to grab a change specification template.
1540 This does not use "p4 -G", as it is nice to keep the submission
1541 template in original order, since a human might edit it.
1543 Remove lines in the Files section that show changes to files
1544 outside the depot path we're committing into."""
1546 [upstream, settings] = findUpstreamBranchPoint()
1549 # A Perforce Change Specification.
1551 # Change: The change number. 'new' on a new changelist.
1552 # Date: The date this specification was last modified.
1553 # Client: The client on which the changelist was created. Read-only.
1554 # User: The user who created the changelist.
1555 # Status: Either 'pending' or 'submitted'. Read-only.
1556 # Type: Either 'public' or 'restricted'. Default is 'public'.
1557 # Description: Comments about the changelist. Required.
1558 # Jobs: What opened jobs are to be closed by this changelist.
1559 # You may delete jobs from this list. (New changelists only.)
1560 # Files: What opened files from the default changelist are to be added
1561 # to this changelist. You may delete files from this list.
1562 # (New changelists only.)
1565 inFilesSection = False
1567 args = ['change', '-o']
1569 args.append(str(changelist))
1570 for entry in p4CmdList(args):
1571 if not entry.has_key('code'):
1573 if entry['code'] == 'stat':
1574 change_entry = entry
1576 if not change_entry:
1577 die('Failed to decode output of p4 change -o')
1578 for key, value in change_entry.iteritems():
1579 if key.startswith('File'):
1580 if settings.has_key('depot-paths'):
1581 if not [p for p in settings['depot-paths']
1582 if p4PathStartsWith(value, p)]:
1585 if not p4PathStartsWith(value, self.depotPath):
1587 files_list.append(value)
1589 # Output in the order expected by prepareLogMessage
1590 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1591 if not change_entry.has_key(key):
1594 template += key + ':'
1595 if key == 'Description':
1597 for field_line in change_entry[key].splitlines():
1598 template += '\t'+field_line+'\n'
1599 if len(files_list) > 0:
1601 template += 'Files:\n'
1602 for path in files_list:
1603 template += '\t'+path+'\n'
1606 def edit_template(self, template_file):
1607 """Invoke the editor to let the user change the submission
1608 message. Return true if okay to continue with the submit."""
1610 # if configured to skip the editing part, just submit
1611 if gitConfigBool("git-p4.skipSubmitEdit"):
1614 # look at the modification time, to check later if the user saved
1616 mtime = os.stat(template_file).st_mtime
1619 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1620 editor = os.environ.get("P4EDITOR")
1622 editor = read_pipe("git var GIT_EDITOR").strip()
1623 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1625 # If the file was not saved, prompt to see if this patch should
1626 # be skipped. But skip this verification step if configured so.
1627 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1630 # modification time updated means user saved the file
1631 if os.stat(template_file).st_mtime > mtime:
1635 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1641 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1643 if os.environ.has_key("P4DIFF"):
1644 del(os.environ["P4DIFF"])
1646 for editedFile in editedFiles:
1647 diff += p4_read_pipe(['diff', '-du',
1648 wildcard_encode(editedFile)])
1652 for newFile in filesToAdd:
1653 newdiff += "==== new file ====\n"
1654 newdiff += "--- /dev/null\n"
1655 newdiff += "+++ %s\n" % newFile
1657 is_link = os.path.islink(newFile)
1658 expect_link = newFile in symlinks
1660 if is_link and expect_link:
1661 newdiff += "+%s\n" % os.readlink(newFile)
1663 f = open(newFile, "r")
1664 for line in f.readlines():
1665 newdiff += "+" + line
1668 return (diff + newdiff).replace('\r\n', '\n')
1670 def applyCommit(self, id):
1671 """Apply one commit, return True if it succeeded."""
1673 print "Applying", read_pipe(["git", "show", "-s",
1674 "--format=format:%h %s", id])
1676 (p4User, gitEmail) = self.p4UserForCommit(id)
1678 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1680 filesToChangeType = set()
1681 filesToDelete = set()
1683 pureRenameCopy = set()
1685 filesToChangeExecBit = {}
1689 diff = parseDiffTreeEntry(line)
1690 modifier = diff['status']
1692 all_files.append(path)
1696 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1697 filesToChangeExecBit[path] = diff['dst_mode']
1698 editedFiles.add(path)
1699 elif modifier == "A":
1700 filesToAdd.add(path)
1701 filesToChangeExecBit[path] = diff['dst_mode']
1702 if path in filesToDelete:
1703 filesToDelete.remove(path)
1705 dst_mode = int(diff['dst_mode'], 8)
1706 if dst_mode == 0120000:
1709 elif modifier == "D":
1710 filesToDelete.add(path)
1711 if path in filesToAdd:
1712 filesToAdd.remove(path)
1713 elif modifier == "C":
1714 src, dest = diff['src'], diff['dst']
1715 p4_integrate(src, dest)
1716 pureRenameCopy.add(dest)
1717 if diff['src_sha1'] != diff['dst_sha1']:
1719 pureRenameCopy.discard(dest)
1720 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1722 pureRenameCopy.discard(dest)
1723 filesToChangeExecBit[dest] = diff['dst_mode']
1725 # turn off read-only attribute
1726 os.chmod(dest, stat.S_IWRITE)
1728 editedFiles.add(dest)
1729 elif modifier == "R":
1730 src, dest = diff['src'], diff['dst']
1731 if self.p4HasMoveCommand:
1732 p4_edit(src) # src must be open before move
1733 p4_move(src, dest) # opens for (move/delete, move/add)
1735 p4_integrate(src, dest)
1736 if diff['src_sha1'] != diff['dst_sha1']:
1739 pureRenameCopy.add(dest)
1740 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1741 if not self.p4HasMoveCommand:
1742 p4_edit(dest) # with move: already open, writable
1743 filesToChangeExecBit[dest] = diff['dst_mode']
1744 if not self.p4HasMoveCommand:
1746 os.chmod(dest, stat.S_IWRITE)
1748 filesToDelete.add(src)
1749 editedFiles.add(dest)
1750 elif modifier == "T":
1751 filesToChangeType.add(path)
1753 die("unknown modifier %s for %s" % (modifier, path))
1755 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1756 patchcmd = diffcmd + " | git apply "
1757 tryPatchCmd = patchcmd + "--check -"
1758 applyPatchCmd = patchcmd + "--check --apply -"
1759 patch_succeeded = True
1761 if os.system(tryPatchCmd) != 0:
1762 fixed_rcs_keywords = False
1763 patch_succeeded = False
1764 print "Unfortunately applying the change failed!"
1766 # Patch failed, maybe it's just RCS keyword woes. Look through
1767 # the patch to see if that's possible.
1768 if gitConfigBool("git-p4.attemptRCSCleanup"):
1772 for file in editedFiles | filesToDelete:
1773 # did this file's delta contain RCS keywords?
1774 pattern = p4_keywords_regexp_for_file(file)
1777 # this file is a possibility...look for RCS keywords.
1778 regexp = re.compile(pattern, re.VERBOSE)
1779 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1780 if regexp.search(line):
1782 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1783 kwfiles[file] = pattern
1786 for file in kwfiles:
1788 print "zapping %s with %s" % (line,pattern)
1789 # File is being deleted, so not open in p4. Must
1790 # disable the read-only bit on windows.
1791 if self.isWindows and file not in editedFiles:
1792 os.chmod(file, stat.S_IWRITE)
1793 self.patchRCSKeywords(file, kwfiles[file])
1794 fixed_rcs_keywords = True
1796 if fixed_rcs_keywords:
1797 print "Retrying the patch with RCS keywords cleaned up"
1798 if os.system(tryPatchCmd) == 0:
1799 patch_succeeded = True
1801 if not patch_succeeded:
1802 for f in editedFiles:
1807 # Apply the patch for real, and do add/delete/+x handling.
1809 system(applyPatchCmd)
1811 for f in filesToChangeType:
1812 p4_edit(f, "-t", "auto")
1813 for f in filesToAdd:
1815 for f in filesToDelete:
1819 # Set/clear executable bits
1820 for f in filesToChangeExecBit.keys():
1821 mode = filesToChangeExecBit[f]
1822 setP4ExecBit(f, mode)
1825 if len(self.update_shelve) > 0:
1826 update_shelve = self.update_shelve.pop(0)
1827 p4_reopen_in_change(update_shelve, all_files)
1830 # Build p4 change description, starting with the contents
1831 # of the git commit message.
1833 logMessage = extractLogMessageFromGitCommit(id)
1834 logMessage = logMessage.strip()
1835 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1837 template = self.prepareSubmitTemplate(update_shelve)
1838 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1840 if self.preserveUser:
1841 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1843 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1844 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1845 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1846 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1848 separatorLine = "######## everything below this line is just the diff #######\n"
1849 if not self.prepare_p4_only:
1850 submitTemplate += separatorLine
1851 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1853 (handle, fileName) = tempfile.mkstemp()
1854 tmpFile = os.fdopen(handle, "w+b")
1856 submitTemplate = submitTemplate.replace("\n", "\r\n")
1857 tmpFile.write(submitTemplate)
1860 if self.prepare_p4_only:
1862 # Leave the p4 tree prepared, and the submit template around
1863 # and let the user decide what to do next
1866 print "P4 workspace prepared for submission."
1867 print "To submit or revert, go to client workspace"
1868 print " " + self.clientPath
1870 print "To submit, use \"p4 submit\" to write a new description,"
1871 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1872 " \"git p4\"." % fileName
1873 print "You can delete the file \"%s\" when finished." % fileName
1875 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1876 print "To preserve change ownership by user %s, you must\n" \
1877 "do \"p4 change -f <change>\" after submitting and\n" \
1878 "edit the User field."
1880 print "After submitting, renamed files must be re-synced."
1881 print "Invoke \"p4 sync -f\" on each of these files:"
1882 for f in pureRenameCopy:
1886 print "To revert the changes, use \"p4 revert ...\", and delete"
1887 print "the submit template file \"%s\"" % fileName
1889 print "Since the commit adds new files, they must be deleted:"
1890 for f in filesToAdd:
1896 # Let the user edit the change description, then submit it.
1901 if self.edit_template(fileName):
1902 # read the edited message and submit
1903 tmpFile = open(fileName, "rb")
1904 message = tmpFile.read()
1907 message = message.replace("\r\n", "\n")
1908 submitTemplate = message[:message.index(separatorLine)]
1911 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1913 p4_write_pipe(['shelve', '-i'], submitTemplate)
1915 p4_write_pipe(['submit', '-i'], submitTemplate)
1916 # The rename/copy happened by applying a patch that created a
1917 # new file. This leaves it writable, which confuses p4.
1918 for f in pureRenameCopy:
1921 if self.preserveUser:
1923 # Get last changelist number. Cannot easily get it from
1924 # the submit command output as the output is
1926 changelist = self.lastP4Changelist()
1927 self.modifyChangelistUser(changelist, p4User)
1933 if not submitted or self.shelve:
1935 print ("Reverting shelved files.")
1937 print ("Submission cancelled, undoing p4 changes.")
1938 for f in editedFiles | filesToDelete:
1940 for f in filesToAdd:
1947 # Export git tags as p4 labels. Create a p4 label and then tag
1949 def exportGitTags(self, gitTags):
1950 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1951 if len(validLabelRegexp) == 0:
1952 validLabelRegexp = defaultLabelRegexp
1953 m = re.compile(validLabelRegexp)
1955 for name in gitTags:
1957 if not m.match(name):
1959 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1962 # Get the p4 commit this corresponds to
1963 logMessage = extractLogMessageFromGitCommit(name)
1964 values = extractSettingsGitLog(logMessage)
1966 if not values.has_key('change'):
1967 # a tag pointing to something not sent to p4; ignore
1969 print "git tag %s does not give a p4 commit" % name
1972 changelist = values['change']
1974 # Get the tag details.
1978 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1981 if re.match(r'tag\s+', l):
1983 elif re.match(r'\s*$', l):
1990 body = ["lightweight tag imported by git p4\n"]
1992 # Create the label - use the same view as the client spec we are using
1993 clientSpec = getClientSpec()
1995 labelTemplate = "Label: %s\n" % name
1996 labelTemplate += "Description:\n"
1998 labelTemplate += "\t" + b + "\n"
1999 labelTemplate += "View:\n"
2000 for depot_side in clientSpec.mappings:
2001 labelTemplate += "\t%s\n" % depot_side
2004 print "Would create p4 label %s for tag" % name
2005 elif self.prepare_p4_only:
2006 print "Not creating p4 label %s for tag due to option" \
2007 " --prepare-p4-only" % name
2009 p4_write_pipe(["label", "-i"], labelTemplate)
2012 p4_system(["tag", "-l", name] +
2013 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2016 print "created p4 label for tag %s" % name
2018 def run(self, args):
2020 self.master = currentGitBranch()
2021 elif len(args) == 1:
2022 self.master = args[0]
2023 if not branchExists(self.master):
2024 die("Branch %s does not exist" % self.master)
2028 for i in self.update_shelve:
2030 sys.exit("invalid changelist %d" % i)
2033 allowSubmit = gitConfig("git-p4.allowSubmit")
2034 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2035 die("%s is not in git-p4.allowSubmit" % self.master)
2037 [upstream, settings] = findUpstreamBranchPoint()
2038 self.depotPath = settings['depot-paths'][0]
2039 if len(self.origin) == 0:
2040 self.origin = upstream
2042 if len(self.update_shelve) > 0:
2045 if self.preserveUser:
2046 if not self.canChangeChangelists():
2047 die("Cannot preserve user names without p4 super-user or admin permissions")
2049 # if not set from the command line, try the config file
2050 if self.conflict_behavior is None:
2051 val = gitConfig("git-p4.conflict")
2053 if val not in self.conflict_behavior_choices:
2054 die("Invalid value '%s' for config git-p4.conflict" % val)
2057 self.conflict_behavior = val
2060 print "Origin branch is " + self.origin
2062 if len(self.depotPath) == 0:
2063 print "Internal error: cannot locate perforce depot path from existing branches"
2066 self.useClientSpec = False
2067 if gitConfigBool("git-p4.useclientspec"):
2068 self.useClientSpec = True
2069 if self.useClientSpec:
2070 self.clientSpecDirs = getClientSpec()
2072 # Check for the existence of P4 branches
2073 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2075 if self.useClientSpec and not branchesDetected:
2076 # all files are relative to the client spec
2077 self.clientPath = getClientRoot()
2079 self.clientPath = p4Where(self.depotPath)
2081 if self.clientPath == "":
2082 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2084 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2085 self.oldWorkingDirectory = os.getcwd()
2087 # ensure the clientPath exists
2088 new_client_dir = False
2089 if not os.path.exists(self.clientPath):
2090 new_client_dir = True
2091 os.makedirs(self.clientPath)
2093 chdir(self.clientPath, is_client_path=True)
2095 print "Would synchronize p4 checkout in %s" % self.clientPath
2097 print "Synchronizing p4 checkout..."
2099 # old one was destroyed, and maybe nobody told p4
2100 p4_sync("...", "-f")
2107 commitish = self.master
2111 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2112 commits.append(line.strip())
2115 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2116 self.checkAuthorship = False
2118 self.checkAuthorship = True
2120 if self.preserveUser:
2121 self.checkValidP4Users(commits)
2124 # Build up a set of options to be passed to diff when
2125 # submitting each commit to p4.
2127 if self.detectRenames:
2128 # command-line -M arg
2129 self.diffOpts = "-M"
2131 # If not explicitly set check the config variable
2132 detectRenames = gitConfig("git-p4.detectRenames")
2134 if detectRenames.lower() == "false" or detectRenames == "":
2136 elif detectRenames.lower() == "true":
2137 self.diffOpts = "-M"
2139 self.diffOpts = "-M%s" % detectRenames
2141 # no command-line arg for -C or --find-copies-harder, just
2143 detectCopies = gitConfig("git-p4.detectCopies")
2144 if detectCopies.lower() == "false" or detectCopies == "":
2146 elif detectCopies.lower() == "true":
2147 self.diffOpts += " -C"
2149 self.diffOpts += " -C%s" % detectCopies
2151 if gitConfigBool("git-p4.detectCopiesHarder"):
2152 self.diffOpts += " --find-copies-harder"
2154 num_shelves = len(self.update_shelve)
2155 if num_shelves > 0 and num_shelves != len(commits):
2156 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2157 (len(commits), num_shelves))
2160 # Apply the commits, one at a time. On failure, ask if should
2161 # continue to try the rest of the patches, or quit.
2166 last = len(commits) - 1
2167 for i, commit in enumerate(commits):
2169 print " ", read_pipe(["git", "show", "-s",
2170 "--format=format:%h %s", commit])
2173 ok = self.applyCommit(commit)
2175 applied.append(commit)
2177 if self.prepare_p4_only and i < last:
2178 print "Processing only the first commit due to option" \
2179 " --prepare-p4-only"
2184 # prompt for what to do, or use the option/variable
2185 if self.conflict_behavior == "ask":
2186 print "What do you want to do?"
2187 response = raw_input("[s]kip this commit but apply"
2188 " the rest, or [q]uit? ")
2191 elif self.conflict_behavior == "skip":
2193 elif self.conflict_behavior == "quit":
2196 die("Unknown conflict_behavior '%s'" %
2197 self.conflict_behavior)
2199 if response[0] == "s":
2200 print "Skipping this commit, but applying the rest"
2202 if response[0] == "q":
2209 chdir(self.oldWorkingDirectory)
2210 shelved_applied = "shelved" if self.shelve else "applied"
2213 elif self.prepare_p4_only:
2215 elif len(commits) == len(applied):
2216 print ("All commits {0}!".format(shelved_applied))
2220 sync.branch = self.branch
2227 if len(applied) == 0:
2228 print ("No commits {0}.".format(shelved_applied))
2230 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2236 print star, read_pipe(["git", "show", "-s",
2237 "--format=format:%h %s", c])
2238 print "You will have to do 'git p4 sync' and rebase."
2240 if gitConfigBool("git-p4.exportLabels"):
2241 self.exportLabels = True
2243 if self.exportLabels:
2244 p4Labels = getP4Labels(self.depotPath)
2245 gitTags = getGitTags()
2247 missingGitTags = gitTags - p4Labels
2248 self.exportGitTags(missingGitTags)
2250 # exit with error unless everything applied perfectly
2251 if len(commits) != len(applied):
2257 """Represent a p4 view ("p4 help views"), and map files in a
2258 repo according to the view."""
2260 def __init__(self, client_name):
2262 self.client_prefix = "//%s/" % client_name
2263 # cache results of "p4 where" to lookup client file locations
2264 self.client_spec_path_cache = {}
2266 def append(self, view_line):
2267 """Parse a view line, splitting it into depot and client
2268 sides. Append to self.mappings, preserving order. This
2269 is only needed for tag creation."""
2271 # Split the view line into exactly two words. P4 enforces
2272 # structure on these lines that simplifies this quite a bit.
2274 # Either or both words may be double-quoted.
2275 # Single quotes do not matter.
2276 # Double-quote marks cannot occur inside the words.
2277 # A + or - prefix is also inside the quotes.
2278 # There are no quotes unless they contain a space.
2279 # The line is already white-space stripped.
2280 # The two words are separated by a single space.
2282 if view_line[0] == '"':
2283 # First word is double quoted. Find its end.
2284 close_quote_index = view_line.find('"', 1)
2285 if close_quote_index <= 0:
2286 die("No first-word closing quote found: %s" % view_line)
2287 depot_side = view_line[1:close_quote_index]
2288 # skip closing quote and space
2289 rhs_index = close_quote_index + 1 + 1
2291 space_index = view_line.find(" ")
2292 if space_index <= 0:
2293 die("No word-splitting space found: %s" % view_line)
2294 depot_side = view_line[0:space_index]
2295 rhs_index = space_index + 1
2297 # prefix + means overlay on previous mapping
2298 if depot_side.startswith("+"):
2299 depot_side = depot_side[1:]
2301 # prefix - means exclude this path, leave out of mappings
2303 if depot_side.startswith("-"):
2305 depot_side = depot_side[1:]
2308 self.mappings.append(depot_side)
2310 def convert_client_path(self, clientFile):
2311 # chop off //client/ part to make it relative
2312 if not clientFile.startswith(self.client_prefix):
2313 die("No prefix '%s' on clientFile '%s'" %
2314 (self.client_prefix, clientFile))
2315 return clientFile[len(self.client_prefix):]
2317 def update_client_spec_path_cache(self, files):
2318 """ Caching file paths by "p4 where" batch query """
2320 # List depot file paths exclude that already cached
2321 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2323 if len(fileArgs) == 0:
2324 return # All files in cache
2326 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2327 for res in where_result:
2328 if "code" in res and res["code"] == "error":
2329 # assume error is "... file(s) not in client view"
2331 if "clientFile" not in res:
2332 die("No clientFile in 'p4 where' output")
2334 # it will list all of them, but only one not unmap-ped
2336 if gitConfigBool("core.ignorecase"):
2337 res['depotFile'] = res['depotFile'].lower()
2338 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2340 # not found files or unmap files set to ""
2341 for depotFile in fileArgs:
2342 if gitConfigBool("core.ignorecase"):
2343 depotFile = depotFile.lower()
2344 if depotFile not in self.client_spec_path_cache:
2345 self.client_spec_path_cache[depotFile] = ""
2347 def map_in_client(self, depot_path):
2348 """Return the relative location in the client where this
2349 depot file should live. Returns "" if the file should
2350 not be mapped in the client."""
2352 if gitConfigBool("core.ignorecase"):
2353 depot_path = depot_path.lower()
2355 if depot_path in self.client_spec_path_cache:
2356 return self.client_spec_path_cache[depot_path]
2358 die( "Error: %s is not found in client spec path" % depot_path )
2361 class P4Sync(Command, P4UserMap):
2362 delete_actions = ( "delete", "move/delete", "purge" )
2365 Command.__init__(self)
2366 P4UserMap.__init__(self)
2368 optparse.make_option("--branch", dest="branch"),
2369 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2370 optparse.make_option("--changesfile", dest="changesFile"),
2371 optparse.make_option("--silent", dest="silent", action="store_true"),
2372 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2373 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2374 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2375 help="Import into refs/heads/ , not refs/remotes"),
2376 optparse.make_option("--max-changes", dest="maxChanges",
2377 help="Maximum number of changes to import"),
2378 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2379 help="Internal block size to use when iteratively calling p4 changes"),
2380 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2381 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2382 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2383 help="Only sync files that are included in the Perforce Client Spec"),
2384 optparse.make_option("-/", dest="cloneExclude",
2385 action="append", type="string",
2386 help="exclude depot path"),
2388 self.description = """Imports from Perforce into a git repository.\n
2390 //depot/my/project/ -- to import the current head
2391 //depot/my/project/@all -- to import everything
2392 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2394 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2396 self.usage += " //depot/path[@revRange]"
2398 self.createdBranches = set()
2399 self.committedChanges = set()
2401 self.detectBranches = False
2402 self.detectLabels = False
2403 self.importLabels = False
2404 self.changesFile = ""
2405 self.syncWithOrigin = True
2406 self.importIntoRemotes = True
2407 self.maxChanges = ""
2408 self.changes_block_size = None
2409 self.keepRepoPath = False
2410 self.depotPaths = None
2411 self.p4BranchesInGit = []
2412 self.cloneExclude = []
2413 self.useClientSpec = False
2414 self.useClientSpec_from_options = False
2415 self.clientSpecDirs = None
2416 self.tempBranches = []
2417 self.tempBranchLocation = "refs/git-p4-tmp"
2418 self.largeFileSystem = None
2420 if gitConfig('git-p4.largeFileSystem'):
2421 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2422 self.largeFileSystem = largeFileSystemConstructor(
2423 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2426 if gitConfig("git-p4.syncFromOrigin") == "false":
2427 self.syncWithOrigin = False
2429 self.depotPaths = []
2430 self.changeRange = ""
2431 self.previousDepotPaths = []
2432 self.hasOrigin = False
2434 # map from branch depot path to parent branch
2435 self.knownBranches = {}
2436 self.initialParents = {}
2438 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2441 # Force a checkpoint in fast-import and wait for it to finish
2442 def checkpoint(self):
2443 self.gitStream.write("checkpoint\n\n")
2444 self.gitStream.write("progress checkpoint\n\n")
2445 out = self.gitOutput.readline()
2447 print "checkpoint finished: " + out
2449 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2450 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2451 for path in self.cloneExclude]
2454 while commit.has_key("depotFile%s" % fnum):
2455 path = commit["depotFile%s" % fnum]
2457 if [p for p in self.cloneExclude
2458 if p4PathStartsWith(path, p)]:
2461 found = [p for p in self.depotPaths
2462 if p4PathStartsWith(path, p)]
2469 file["rev"] = commit["rev%s" % fnum]
2470 file["action"] = commit["action%s" % fnum]
2471 file["type"] = commit["type%s" % fnum]
2473 file["shelved_cl"] = int(shelved_cl)
2479 def extractJobsFromCommit(self, commit):
2482 while commit.has_key("job%s" % jnum):
2483 job = commit["job%s" % jnum]
2488 def stripRepoPath(self, path, prefixes):
2489 """When streaming files, this is called to map a p4 depot path
2490 to where it should go in git. The prefixes are either
2491 self.depotPaths, or self.branchPrefixes in the case of
2492 branch detection."""
2494 if self.useClientSpec:
2495 # branch detection moves files up a level (the branch name)
2496 # from what client spec interpretation gives
2497 path = self.clientSpecDirs.map_in_client(path)
2498 if self.detectBranches:
2499 for b in self.knownBranches:
2500 if path.startswith(b + "/"):
2501 path = path[len(b)+1:]
2503 elif self.keepRepoPath:
2504 # Preserve everything in relative path name except leading
2505 # //depot/; just look at first prefix as they all should
2506 # be in the same depot.
2507 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2508 if p4PathStartsWith(path, depot):
2509 path = path[len(depot):]
2513 if p4PathStartsWith(path, p):
2514 path = path[len(p):]
2517 path = wildcard_decode(path)
2520 def splitFilesIntoBranches(self, commit):
2521 """Look at each depotFile in the commit to figure out to what
2522 branch it belongs."""
2524 if self.clientSpecDirs:
2525 files = self.extractFilesFromCommit(commit)
2526 self.clientSpecDirs.update_client_spec_path_cache(files)
2530 while commit.has_key("depotFile%s" % fnum):
2531 path = commit["depotFile%s" % fnum]
2532 found = [p for p in self.depotPaths
2533 if p4PathStartsWith(path, p)]
2540 file["rev"] = commit["rev%s" % fnum]
2541 file["action"] = commit["action%s" % fnum]
2542 file["type"] = commit["type%s" % fnum]
2545 # start with the full relative path where this file would
2547 if self.useClientSpec:
2548 relPath = self.clientSpecDirs.map_in_client(path)
2550 relPath = self.stripRepoPath(path, self.depotPaths)
2552 for branch in self.knownBranches.keys():
2553 # add a trailing slash so that a commit into qt/4.2foo
2554 # doesn't end up in qt/4.2, e.g.
2555 if relPath.startswith(branch + "/"):
2556 if branch not in branches:
2557 branches[branch] = []
2558 branches[branch].append(file)
2563 def writeToGitStream(self, gitMode, relPath, contents):
2564 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2565 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2567 self.gitStream.write(d)
2568 self.gitStream.write('\n')
2570 def encodeWithUTF8(self, path):
2572 path.decode('ascii')
2575 if gitConfig('git-p4.pathEncoding'):
2576 encoding = gitConfig('git-p4.pathEncoding')
2577 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2579 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2582 # output one file from the P4 stream
2583 # - helper for streamP4Files
2585 def streamOneP4File(self, file, contents):
2586 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2587 relPath = self.encodeWithUTF8(relPath)
2589 size = int(self.stream_file['fileSize'])
2590 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2593 (type_base, type_mods) = split_p4_type(file["type"])
2596 if "x" in type_mods:
2598 if type_base == "symlink":
2600 # p4 print on a symlink sometimes contains "target\n";
2601 # if it does, remove the newline
2602 data = ''.join(contents)
2604 # Some version of p4 allowed creating a symlink that pointed
2605 # to nothing. This causes p4 errors when checking out such
2606 # a change, and errors here too. Work around it by ignoring
2607 # the bad symlink; hopefully a future change fixes it.
2608 print "\nIgnoring empty symlink in %s" % file['depotFile']
2610 elif data[-1] == '\n':
2611 contents = [data[:-1]]
2615 if type_base == "utf16":
2616 # p4 delivers different text in the python output to -G
2617 # than it does when using "print -o", or normal p4 client
2618 # operations. utf16 is converted to ascii or utf8, perhaps.
2619 # But ascii text saved as -t utf16 is completely mangled.
2620 # Invoke print -o to get the real contents.
2622 # On windows, the newlines will always be mangled by print, so put
2623 # them back too. This is not needed to the cygwin windows version,
2624 # just the native "NT" type.
2627 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2628 except Exception as e:
2629 if 'Translation of file content failed' in str(e):
2630 type_base = 'binary'
2634 if p4_version_string().find('/NT') >= 0:
2635 text = text.replace('\r\n', '\n')
2638 if type_base == "apple":
2639 # Apple filetype files will be streamed as a concatenation of
2640 # its appledouble header and the contents. This is useless
2641 # on both macs and non-macs. If using "print -q -o xx", it
2642 # will create "xx" with the data, and "%xx" with the header.
2643 # This is also not very useful.
2645 # Ideally, someday, this script can learn how to generate
2646 # appledouble files directly and import those to git, but
2647 # non-mac machines can never find a use for apple filetype.
2648 print "\nIgnoring apple filetype file %s" % file['depotFile']
2651 # Note that we do not try to de-mangle keywords on utf16 files,
2652 # even though in theory somebody may want that.
2653 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2655 regexp = re.compile(pattern, re.VERBOSE)
2656 text = ''.join(contents)
2657 text = regexp.sub(r'$\1$', text)
2660 if self.largeFileSystem:
2661 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2663 self.writeToGitStream(git_mode, relPath, contents)
2665 def streamOneP4Deletion(self, file):
2666 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2667 relPath = self.encodeWithUTF8(relPath)
2669 sys.stdout.write("delete %s\n" % relPath)
2671 self.gitStream.write("D %s\n" % relPath)
2673 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2674 self.largeFileSystem.removeLargeFile(relPath)
2676 # handle another chunk of streaming data
2677 def streamP4FilesCb(self, marshalled):
2679 # catch p4 errors and complain
2681 if "code" in marshalled:
2682 if marshalled["code"] == "error":
2683 if "data" in marshalled:
2684 err = marshalled["data"].rstrip()
2686 if not err and 'fileSize' in self.stream_file:
2687 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2688 if required_bytes > 0:
2689 err = 'Not enough space left on %s! Free at least %i MB.' % (
2690 os.getcwd(), required_bytes/1024/1024
2695 if self.stream_have_file_info:
2696 if "depotFile" in self.stream_file:
2697 f = self.stream_file["depotFile"]
2698 # force a failure in fast-import, else an empty
2699 # commit will be made
2700 self.gitStream.write("\n")
2701 self.gitStream.write("die-now\n")
2702 self.gitStream.close()
2703 # ignore errors, but make sure it exits first
2704 self.importProcess.wait()
2706 die("Error from p4 print for %s: %s" % (f, err))
2708 die("Error from p4 print: %s" % err)
2710 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2711 # start of a new file - output the old one first
2712 self.streamOneP4File(self.stream_file, self.stream_contents)
2713 self.stream_file = {}
2714 self.stream_contents = []
2715 self.stream_have_file_info = False
2717 # pick up the new file information... for the
2718 # 'data' field we need to append to our array
2719 for k in marshalled.keys():
2721 if 'streamContentSize' not in self.stream_file:
2722 self.stream_file['streamContentSize'] = 0
2723 self.stream_file['streamContentSize'] += len(marshalled['data'])
2724 self.stream_contents.append(marshalled['data'])
2726 self.stream_file[k] = marshalled[k]
2729 'streamContentSize' in self.stream_file and
2730 'fileSize' in self.stream_file and
2731 'depotFile' in self.stream_file):
2732 size = int(self.stream_file["fileSize"])
2734 progress = 100*self.stream_file['streamContentSize']/size
2735 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2738 self.stream_have_file_info = True
2740 # Stream directly from "p4 files" into "git fast-import"
2741 def streamP4Files(self, files):
2747 filesForCommit.append(f)
2748 if f['action'] in self.delete_actions:
2749 filesToDelete.append(f)
2751 filesToRead.append(f)
2754 for f in filesToDelete:
2755 self.streamOneP4Deletion(f)
2757 if len(filesToRead) > 0:
2758 self.stream_file = {}
2759 self.stream_contents = []
2760 self.stream_have_file_info = False
2762 # curry self argument
2763 def streamP4FilesCbSelf(entry):
2764 self.streamP4FilesCb(entry)
2767 for f in filesToRead:
2768 if 'shelved_cl' in f:
2769 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2771 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2773 fileArg = '%s#%s' % (f['path'], f['rev'])
2775 fileArgs.append(fileArg)
2777 p4CmdList(["-x", "-", "print"],
2779 cb=streamP4FilesCbSelf)
2782 if self.stream_file.has_key('depotFile'):
2783 self.streamOneP4File(self.stream_file, self.stream_contents)
2785 def make_email(self, userid):
2786 if userid in self.users:
2787 return self.users[userid]
2789 return "%s <a@b>" % userid
2791 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2792 """ Stream a p4 tag.
2793 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2797 print "writing tag %s for commit %s" % (labelName, commit)
2798 gitStream.write("tag %s\n" % labelName)
2799 gitStream.write("from %s\n" % commit)
2801 if labelDetails.has_key('Owner'):
2802 owner = labelDetails["Owner"]
2806 # Try to use the owner of the p4 label, or failing that,
2807 # the current p4 user id.
2809 email = self.make_email(owner)
2811 email = self.make_email(self.p4UserId())
2812 tagger = "%s %s %s" % (email, epoch, self.tz)
2814 gitStream.write("tagger %s\n" % tagger)
2816 print "labelDetails=",labelDetails
2817 if labelDetails.has_key('Description'):
2818 description = labelDetails['Description']
2820 description = 'Label from git p4'
2822 gitStream.write("data %d\n" % len(description))
2823 gitStream.write(description)
2824 gitStream.write("\n")
2826 def inClientSpec(self, path):
2827 if not self.clientSpecDirs:
2829 inClientSpec = self.clientSpecDirs.map_in_client(path)
2830 if not inClientSpec and self.verbose:
2831 print('Ignoring file outside of client spec: {0}'.format(path))
2834 def hasBranchPrefix(self, path):
2835 if not self.branchPrefixes:
2837 hasPrefix = [p for p in self.branchPrefixes
2838 if p4PathStartsWith(path, p)]
2839 if not hasPrefix and self.verbose:
2840 print('Ignoring file outside of prefix: {0}'.format(path))
2843 def commit(self, details, files, branch, parent = ""):
2844 epoch = details["time"]
2845 author = details["user"]
2846 jobs = self.extractJobsFromCommit(details)
2849 print('commit into {0}'.format(branch))
2851 if self.clientSpecDirs:
2852 self.clientSpecDirs.update_client_spec_path_cache(files)
2854 files = [f for f in files
2855 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2857 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2858 print('Ignoring revision {0} as it would produce an empty commit.'
2859 .format(details['change']))
2862 self.gitStream.write("commit %s\n" % branch)
2863 self.gitStream.write("mark :%s\n" % details["change"])
2864 self.committedChanges.add(int(details["change"]))
2866 if author not in self.users:
2867 self.getUserMapFromPerforceServer()
2868 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2870 self.gitStream.write("committer %s\n" % committer)
2872 self.gitStream.write("data <<EOT\n")
2873 self.gitStream.write(details["desc"])
2875 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2876 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2877 (','.join(self.branchPrefixes), details["change"]))
2878 if len(details['options']) > 0:
2879 self.gitStream.write(": options = %s" % details['options'])
2880 self.gitStream.write("]\nEOT\n\n")
2884 print "parent %s" % parent
2885 self.gitStream.write("from %s\n" % parent)
2887 self.streamP4Files(files)
2888 self.gitStream.write("\n")
2890 change = int(details["change"])
2892 if self.labels.has_key(change):
2893 label = self.labels[change]
2894 labelDetails = label[0]
2895 labelRevisions = label[1]
2897 print "Change %s is labelled %s" % (change, labelDetails)
2899 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2900 for p in self.branchPrefixes])
2902 if len(files) == len(labelRevisions):
2906 if info["action"] in self.delete_actions:
2908 cleanedFiles[info["depotFile"]] = info["rev"]
2910 if cleanedFiles == labelRevisions:
2911 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2915 print ("Tag %s does not match with change %s: files do not match."
2916 % (labelDetails["label"], change))
2920 print ("Tag %s does not match with change %s: file count is different."
2921 % (labelDetails["label"], change))
2923 # Build a dictionary of changelists and labels, for "detect-labels" option.
2924 def getLabels(self):
2927 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2928 if len(l) > 0 and not self.silent:
2929 print "Finding files belonging to labels in %s" % `self.depotPaths`
2932 label = output["label"]
2936 print "Querying files for label %s" % label
2937 for file in p4CmdList(["files"] +
2938 ["%s...@%s" % (p, label)
2939 for p in self.depotPaths]):
2940 revisions[file["depotFile"]] = file["rev"]
2941 change = int(file["change"])
2942 if change > newestChange:
2943 newestChange = change
2945 self.labels[newestChange] = [output, revisions]
2948 print "Label changes: %s" % self.labels.keys()
2950 # Import p4 labels as git tags. A direct mapping does not
2951 # exist, so assume that if all the files are at the same revision
2952 # then we can use that, or it's something more complicated we should
2954 def importP4Labels(self, stream, p4Labels):
2956 print "import p4 labels: " + ' '.join(p4Labels)
2958 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2959 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2960 if len(validLabelRegexp) == 0:
2961 validLabelRegexp = defaultLabelRegexp
2962 m = re.compile(validLabelRegexp)
2964 for name in p4Labels:
2967 if not m.match(name):
2969 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2972 if name in ignoredP4Labels:
2975 labelDetails = p4CmdList(['label', "-o", name])[0]
2977 # get the most recent changelist for each file in this label
2978 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2979 for p in self.depotPaths])
2981 if change.has_key('change'):
2982 # find the corresponding git commit; take the oldest commit
2983 changelist = int(change['change'])
2984 if changelist in self.committedChanges:
2985 gitCommit = ":%d" % changelist # use a fast-import mark
2988 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2989 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2990 if len(gitCommit) == 0:
2991 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2994 gitCommit = gitCommit.strip()
2997 # Convert from p4 time format
2999 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3001 print "Could not convert label time %s" % labelDetails['Update']
3004 when = int(time.mktime(tmwhen))
3005 self.streamTag(stream, name, labelDetails, gitCommit, when)
3007 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3010 print "Label %s has no changelists - possibly deleted?" % name
3013 # We can't import this label; don't try again as it will get very
3014 # expensive repeatedly fetching all the files for labels that will
3015 # never be imported. If the label is moved in the future, the
3016 # ignore will need to be removed manually.
3017 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3019 def guessProjectName(self):
3020 for p in self.depotPaths:
3023 p = p[p.strip().rfind("/") + 1:]
3024 if not p.endswith("/"):
3028 def getBranchMapping(self):
3029 lostAndFoundBranches = set()
3031 user = gitConfig("git-p4.branchUser")
3033 command = "branches -u %s" % user
3035 command = "branches"
3037 for info in p4CmdList(command):
3038 details = p4Cmd(["branch", "-o", info["branch"]])
3040 while details.has_key("View%s" % viewIdx):
3041 paths = details["View%s" % viewIdx].split(" ")
3042 viewIdx = viewIdx + 1
3043 # require standard //depot/foo/... //depot/bar/... mapping
3044 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3047 destination = paths[1]
3049 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3050 source = source[len(self.depotPaths[0]):-4]
3051 destination = destination[len(self.depotPaths[0]):-4]
3053 if destination in self.knownBranches:
3055 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3056 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3059 self.knownBranches[destination] = source
3061 lostAndFoundBranches.discard(destination)
3063 if source not in self.knownBranches:
3064 lostAndFoundBranches.add(source)
3066 # Perforce does not strictly require branches to be defined, so we also
3067 # check git config for a branch list.
3069 # Example of branch definition in git config file:
3071 # branchList=main:branchA
3072 # branchList=main:branchB
3073 # branchList=branchA:branchC
3074 configBranches = gitConfigList("git-p4.branchList")
3075 for branch in configBranches:
3077 (source, destination) = branch.split(":")
3078 self.knownBranches[destination] = source
3080 lostAndFoundBranches.discard(destination)
3082 if source not in self.knownBranches:
3083 lostAndFoundBranches.add(source)
3086 for branch in lostAndFoundBranches:
3087 self.knownBranches[branch] = branch
3089 def getBranchMappingFromGitBranches(self):
3090 branches = p4BranchesInGit(self.importIntoRemotes)
3091 for branch in branches.keys():
3092 if branch == "master":
3095 branch = branch[len(self.projectName):]
3096 self.knownBranches[branch] = branch
3098 def updateOptionDict(self, d):
3100 if self.keepRepoPath:
3101 option_keys['keepRepoPath'] = 1
3103 d["options"] = ' '.join(sorted(option_keys.keys()))
3105 def readOptions(self, d):
3106 self.keepRepoPath = (d.has_key('options')
3107 and ('keepRepoPath' in d['options']))
3109 def gitRefForBranch(self, branch):
3110 if branch == "main":
3111 return self.refPrefix + "master"
3113 if len(branch) <= 0:
3116 return self.refPrefix + self.projectName + branch
3118 def gitCommitByP4Change(self, ref, change):
3120 print "looking in ref " + ref + " for change %s using bisect..." % change
3123 latestCommit = parseRevision(ref)
3127 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3128 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3133 log = extractLogMessageFromGitCommit(next)
3134 settings = extractSettingsGitLog(log)
3135 currentChange = int(settings['change'])
3137 print "current change %s" % currentChange
3139 if currentChange == change:
3141 print "found %s" % next
3144 if currentChange < change:
3145 earliestCommit = "^%s" % next
3147 latestCommit = "%s" % next
3151 def importNewBranch(self, branch, maxChange):
3152 # make fast-import flush all changes to disk and update the refs using the checkpoint
3153 # command so that we can try to find the branch parent in the git history
3154 self.gitStream.write("checkpoint\n\n");
3155 self.gitStream.flush();
3156 branchPrefix = self.depotPaths[0] + branch + "/"
3157 range = "@1,%s" % maxChange
3158 #print "prefix" + branchPrefix
3159 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3160 if len(changes) <= 0:
3162 firstChange = changes[0]
3163 #print "first change in branch: %s" % firstChange
3164 sourceBranch = self.knownBranches[branch]
3165 sourceDepotPath = self.depotPaths[0] + sourceBranch
3166 sourceRef = self.gitRefForBranch(sourceBranch)
3167 #print "source " + sourceBranch
3169 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3170 #print "branch parent: %s" % branchParentChange
3171 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3172 if len(gitParent) > 0:
3173 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3174 #print "parent git commit: %s" % gitParent
3176 self.importChanges(changes)
3179 def searchParent(self, parent, branch, target):
3181 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3182 "--no-merges", parent]):
3184 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3187 print "Found parent of %s in commit %s" % (branch, blob)
3194 def importChanges(self, changes, shelved=False):
3196 for change in changes:
3197 description = p4_describe(change, shelved)
3198 self.updateOptionDict(description)
3201 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3206 if self.detectBranches:
3207 branches = self.splitFilesIntoBranches(description)
3208 for branch in branches.keys():
3210 branchPrefix = self.depotPaths[0] + branch + "/"
3211 self.branchPrefixes = [ branchPrefix ]
3215 filesForCommit = branches[branch]
3218 print "branch is %s" % branch
3220 self.updatedBranches.add(branch)
3222 if branch not in self.createdBranches:
3223 self.createdBranches.add(branch)
3224 parent = self.knownBranches[branch]
3225 if parent == branch:
3228 fullBranch = self.projectName + branch
3229 if fullBranch not in self.p4BranchesInGit:
3231 print("\n Importing new branch %s" % fullBranch);
3232 if self.importNewBranch(branch, change - 1):
3234 self.p4BranchesInGit.append(fullBranch)
3236 print("\n Resuming with change %s" % change);
3239 print "parent determined through known branches: %s" % parent
3241 branch = self.gitRefForBranch(branch)
3242 parent = self.gitRefForBranch(parent)
3245 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3247 if len(parent) == 0 and branch in self.initialParents:
3248 parent = self.initialParents[branch]
3249 del self.initialParents[branch]
3253 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3255 print "Creating temporary branch: " + tempBranch
3256 self.commit(description, filesForCommit, tempBranch)
3257 self.tempBranches.append(tempBranch)
3259 blob = self.searchParent(parent, branch, tempBranch)
3261 self.commit(description, filesForCommit, branch, blob)
3264 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3265 self.commit(description, filesForCommit, branch, parent)
3267 files = self.extractFilesFromCommit(description, shelved, change)
3268 self.commit(description, files, self.branch,
3270 # only needed once, to connect to the previous commit
3271 self.initialParent = ""
3273 print self.gitError.read()
3276 def importHeadRevision(self, revision):
3277 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3280 details["user"] = "git perforce import user"
3281 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3282 % (' '.join(self.depotPaths), revision))
3283 details["change"] = revision
3287 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3289 for info in p4CmdList(["files"] + fileArgs):
3291 if 'code' in info and info['code'] == 'error':
3292 sys.stderr.write("p4 returned an error: %s\n"
3294 if info['data'].find("must refer to client") >= 0:
3295 sys.stderr.write("This particular p4 error is misleading.\n")
3296 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3297 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3299 if 'p4ExitCode' in info:
3300 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3304 change = int(info["change"])
3305 if change > newestRevision:
3306 newestRevision = change
3308 if info["action"] in self.delete_actions:
3309 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3310 #fileCnt = fileCnt + 1
3313 for prop in ["depotFile", "rev", "action", "type" ]:
3314 details["%s%s" % (prop, fileCnt)] = info[prop]
3316 fileCnt = fileCnt + 1
3318 details["change"] = newestRevision
3320 # Use time from top-most change so that all git p4 clones of
3321 # the same p4 repo have the same commit SHA1s.
3322 res = p4_describe(newestRevision)
3323 details["time"] = res["time"]
3325 self.updateOptionDict(details)
3327 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3329 print "IO error with git fast-import. Is your git version recent enough?"
3330 print self.gitError.read()
3332 def openStreams(self):
3333 self.importProcess = subprocess.Popen(["git", "fast-import"],
3334 stdin=subprocess.PIPE,
3335 stdout=subprocess.PIPE,
3336 stderr=subprocess.PIPE);
3337 self.gitOutput = self.importProcess.stdout
3338 self.gitStream = self.importProcess.stdin
3339 self.gitError = self.importProcess.stderr
3341 def closeStreams(self):
3342 self.gitStream.close()
3343 if self.importProcess.wait() != 0:
3344 die("fast-import failed: %s" % self.gitError.read())
3345 self.gitOutput.close()
3346 self.gitError.close()
3348 def run(self, args):
3349 if self.importIntoRemotes:
3350 self.refPrefix = "refs/remotes/p4/"
3352 self.refPrefix = "refs/heads/p4/"
3354 if self.syncWithOrigin:
3355 self.hasOrigin = originP4BranchesExist()
3358 print 'Syncing with origin first, using "git fetch origin"'
3359 system("git fetch origin")
3361 branch_arg_given = bool(self.branch)
3362 if len(self.branch) == 0:
3363 self.branch = self.refPrefix + "master"
3364 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3365 system("git update-ref %s refs/heads/p4" % self.branch)
3366 system("git branch -D p4")
3368 # accept either the command-line option, or the configuration variable
3369 if self.useClientSpec:
3370 # will use this after clone to set the variable
3371 self.useClientSpec_from_options = True
3373 if gitConfigBool("git-p4.useclientspec"):
3374 self.useClientSpec = True
3375 if self.useClientSpec:
3376 self.clientSpecDirs = getClientSpec()
3378 # TODO: should always look at previous commits,
3379 # merge with previous imports, if possible.
3382 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3384 # branches holds mapping from branch name to sha1
3385 branches = p4BranchesInGit(self.importIntoRemotes)
3387 # restrict to just this one, disabling detect-branches
3388 if branch_arg_given:
3389 short = self.branch.split("/")[-1]
3390 if short in branches:
3391 self.p4BranchesInGit = [ short ]
3393 self.p4BranchesInGit = branches.keys()
3395 if len(self.p4BranchesInGit) > 1:
3397 print "Importing from/into multiple branches"
3398 self.detectBranches = True
3399 for branch in branches.keys():
3400 self.initialParents[self.refPrefix + branch] = \
3404 print "branches: %s" % self.p4BranchesInGit
3407 for branch in self.p4BranchesInGit:
3408 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3410 settings = extractSettingsGitLog(logMsg)
3412 self.readOptions(settings)
3413 if (settings.has_key('depot-paths')
3414 and settings.has_key ('change')):
3415 change = int(settings['change']) + 1
3416 p4Change = max(p4Change, change)
3418 depotPaths = sorted(settings['depot-paths'])
3419 if self.previousDepotPaths == []:
3420 self.previousDepotPaths = depotPaths
3423 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3424 prev_list = prev.split("/")
3425 cur_list = cur.split("/")
3426 for i in range(0, min(len(cur_list), len(prev_list))):
3427 if cur_list[i] <> prev_list[i]:
3431 paths.append ("/".join(cur_list[:i + 1]))
3433 self.previousDepotPaths = paths
3436 self.depotPaths = sorted(self.previousDepotPaths)
3437 self.changeRange = "@%s,#head" % p4Change
3438 if not self.silent and not self.detectBranches:
3439 print "Performing incremental import into %s git branch" % self.branch
3441 # accept multiple ref name abbreviations:
3442 # refs/foo/bar/branch -> use it exactly
3443 # p4/branch -> prepend refs/remotes/ or refs/heads/
3444 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3445 if not self.branch.startswith("refs/"):
3446 if self.importIntoRemotes:
3447 prepend = "refs/remotes/"
3449 prepend = "refs/heads/"
3450 if not self.branch.startswith("p4/"):
3452 self.branch = prepend + self.branch
3454 if len(args) == 0 and self.depotPaths:
3456 print "Depot paths: %s" % ' '.join(self.depotPaths)
3458 if self.depotPaths and self.depotPaths != args:
3459 print ("previous import used depot path %s and now %s was specified. "
3460 "This doesn't work!" % (' '.join (self.depotPaths),
3464 self.depotPaths = sorted(args)
3469 # Make sure no revision specifiers are used when --changesfile
3471 bad_changesfile = False
3472 if len(self.changesFile) > 0:
3473 for p in self.depotPaths:
3474 if p.find("@") >= 0 or p.find("#") >= 0:
3475 bad_changesfile = True
3478 die("Option --changesfile is incompatible with revision specifiers")
3481 for p in self.depotPaths:
3482 if p.find("@") != -1:
3483 atIdx = p.index("@")
3484 self.changeRange = p[atIdx:]
3485 if self.changeRange == "@all":
3486 self.changeRange = ""
3487 elif ',' not in self.changeRange:
3488 revision = self.changeRange
3489 self.changeRange = ""
3491 elif p.find("#") != -1:
3492 hashIdx = p.index("#")
3493 revision = p[hashIdx:]
3495 elif self.previousDepotPaths == []:
3496 # pay attention to changesfile, if given, else import
3497 # the entire p4 tree at the head revision
3498 if len(self.changesFile) == 0:
3501 p = re.sub ("\.\.\.$", "", p)
3502 if not p.endswith("/"):
3507 self.depotPaths = newPaths
3509 # --detect-branches may change this for each branch
3510 self.branchPrefixes = self.depotPaths
3512 self.loadUserMapFromCache()
3514 if self.detectLabels:
3517 if self.detectBranches:
3518 ## FIXME - what's a P4 projectName ?
3519 self.projectName = self.guessProjectName()
3522 self.getBranchMappingFromGitBranches()
3524 self.getBranchMapping()
3526 print "p4-git branches: %s" % self.p4BranchesInGit
3527 print "initial parents: %s" % self.initialParents
3528 for b in self.p4BranchesInGit:
3532 b = b[len(self.projectName):]
3533 self.createdBranches.add(b)
3538 self.importHeadRevision(revision)
3542 if len(self.changesFile) > 0:
3543 output = open(self.changesFile).readlines()
3546 changeSet.add(int(line))
3548 for change in changeSet:
3549 changes.append(change)
3553 # catch "git p4 sync" with no new branches, in a repo that
3554 # does not have any existing p4 branches
3556 if not self.p4BranchesInGit:
3557 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3559 # The default branch is master, unless --branch is used to
3560 # specify something else. Make sure it exists, or complain
3561 # nicely about how to use --branch.
3562 if not self.detectBranches:
3563 if not branch_exists(self.branch):
3564 if branch_arg_given:
3565 die("Error: branch %s does not exist." % self.branch)
3567 die("Error: no branch %s; perhaps specify one with --branch." %
3571 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3573 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3575 if len(self.maxChanges) > 0:
3576 changes = changes[:min(int(self.maxChanges), len(changes))]
3578 if len(changes) == 0:
3580 print "No changes to import!"
3582 if not self.silent and not self.detectBranches:
3583 print "Import destination: %s" % self.branch
3585 self.updatedBranches = set()
3587 if not self.detectBranches:
3589 # start a new branch
3590 self.initialParent = ""
3592 # build on a previous revision
3593 self.initialParent = parseRevision(self.branch)
3595 self.importChanges(changes)
3599 if len(self.updatedBranches) > 0:
3600 sys.stdout.write("Updated branches: ")
3601 for b in self.updatedBranches:
3602 sys.stdout.write("%s " % b)
3603 sys.stdout.write("\n")
3605 if gitConfigBool("git-p4.importLabels"):
3606 self.importLabels = True
3608 if self.importLabels:
3609 p4Labels = getP4Labels(self.depotPaths)
3610 gitTags = getGitTags()
3612 missingP4Labels = p4Labels - gitTags
3613 self.importP4Labels(self.gitStream, missingP4Labels)
3617 # Cleanup temporary branches created during import
3618 if self.tempBranches != []:
3619 for branch in self.tempBranches:
3620 read_pipe("git update-ref -d %s" % branch)
3621 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3623 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3624 # a convenient shortcut refname "p4".
3625 if self.importIntoRemotes:
3626 head_ref = self.refPrefix + "HEAD"
3627 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3628 system(["git", "symbolic-ref", head_ref, self.branch])
3632 class P4Rebase(Command):
3634 Command.__init__(self)
3636 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3638 self.importLabels = False
3639 self.description = ("Fetches the latest revision from perforce and "
3640 + "rebases the current work (branch) against it")
3642 def run(self, args):
3644 sync.importLabels = self.importLabels
3647 return self.rebase()
3650 if os.system("git update-index --refresh") != 0:
3651 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3652 if len(read_pipe("git diff-index HEAD --")) > 0:
3653 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3655 [upstream, settings] = findUpstreamBranchPoint()
3656 if len(upstream) == 0:
3657 die("Cannot find upstream branchpoint for rebase")
3659 # the branchpoint may be p4/foo~3, so strip off the parent
3660 upstream = re.sub("~[0-9]+$", "", upstream)
3662 print "Rebasing the current branch onto %s" % upstream
3663 oldHead = read_pipe("git rev-parse HEAD").strip()
3664 system("git rebase %s" % upstream)
3665 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3668 class P4Clone(P4Sync):
3670 P4Sync.__init__(self)
3671 self.description = "Creates a new git repository and imports from Perforce into it"
3672 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3674 optparse.make_option("--destination", dest="cloneDestination",
3675 action='store', default=None,
3676 help="where to leave result of the clone"),
3677 optparse.make_option("--bare", dest="cloneBare",
3678 action="store_true", default=False),
3680 self.cloneDestination = None
3681 self.needsGit = False
3682 self.cloneBare = False
3684 def defaultDestination(self, args):
3685 ## TODO: use common prefix of args?
3687 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3688 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3689 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3690 depotDir = re.sub(r"/$", "", depotDir)
3691 return os.path.split(depotDir)[1]
3693 def run(self, args):
3697 if self.keepRepoPath and not self.cloneDestination:
3698 sys.stderr.write("Must specify destination for --keep-path\n")
3703 if not self.cloneDestination and len(depotPaths) > 1:
3704 self.cloneDestination = depotPaths[-1]
3705 depotPaths = depotPaths[:-1]
3707 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3708 for p in depotPaths:
3709 if not p.startswith("//"):
3710 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3713 if not self.cloneDestination:
3714 self.cloneDestination = self.defaultDestination(args)
3716 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3718 if not os.path.exists(self.cloneDestination):
3719 os.makedirs(self.cloneDestination)
3720 chdir(self.cloneDestination)
3722 init_cmd = [ "git", "init" ]
3724 init_cmd.append("--bare")
3725 retcode = subprocess.call(init_cmd)
3727 raise CalledProcessError(retcode, init_cmd)
3729 if not P4Sync.run(self, depotPaths):
3732 # create a master branch and check out a work tree
3733 if gitBranchExists(self.branch):
3734 system([ "git", "branch", "master", self.branch ])
3735 if not self.cloneBare:
3736 system([ "git", "checkout", "-f" ])
3738 print 'Not checking out any branch, use ' \
3739 '"git checkout -q -b master <branch>"'
3741 # auto-set this variable if invoked with --use-client-spec
3742 if self.useClientSpec_from_options:
3743 system("git config --bool git-p4.useclientspec true")
3747 class P4Unshelve(Command):
3749 Command.__init__(self)
3751 self.description = "Unshelve a P4 changelist into a git commit"
3752 self.usage = "usage: %prog [options] changelist"
3754 optparse.make_option("--no-commit", dest="noCommit",
3755 action='store_true', default=False,
3756 help="do not commit, just update the files"),
3757 optparse.make_option("--origin", dest="origin"),
3759 self.verbose = False
3760 self.noCommit = False
3761 self.origin = "p4/master"
3762 self.destbranch = "refs/remotes/p4/unshelved/%s"
3764 def run(self, args):
3768 if not gitBranchExists(self.origin):
3769 sys.exit("origin branch %s does not exist" % self.origin)
3773 sync.initialParent = self.origin
3774 sync.branch = self.destbranch % changes[0]
3775 sync.verbose = self.verbose
3777 log = extractLogMessageFromGitCommit(self.origin)
3778 settings = extractSettingsGitLog(log)
3779 sync.depotPaths = settings['depot-paths']
3780 sync.branchPrefixes = sync.depotPaths
3783 sync.loadUserMapFromCache()
3784 sync.importChanges(changes, shelved=True)
3789 class P4Branches(Command):
3791 Command.__init__(self)
3793 self.description = ("Shows the git branches that hold imports and their "
3794 + "corresponding perforce depot paths")
3795 self.verbose = False
3797 def run(self, args):
3798 if originP4BranchesExist():
3799 createOrUpdateBranchesFromOrigin()
3801 cmdline = "git rev-parse --symbolic "
3802 cmdline += " --remotes"
3804 for line in read_pipe_lines(cmdline):
3807 if not line.startswith('p4/') or line == "p4/HEAD":
3811 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3812 settings = extractSettingsGitLog(log)
3814 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3817 class HelpFormatter(optparse.IndentedHelpFormatter):
3819 optparse.IndentedHelpFormatter.__init__(self)
3821 def format_description(self, description):
3823 return description + "\n"
3827 def printUsage(commands):
3828 print "usage: %s <command> [options]" % sys.argv[0]
3830 print "valid commands: %s" % ", ".join(commands)
3832 print "Try %s <command> --help for command specific help." % sys.argv[0]
3837 "submit" : P4Submit,
3838 "commit" : P4Submit,
3840 "rebase" : P4Rebase,
3842 "rollback" : P4RollBack,
3843 "branches" : P4Branches,
3844 "unshelve" : P4Unshelve,
3849 if len(sys.argv[1:]) == 0:
3850 printUsage(commands.keys())
3853 cmdName = sys.argv[1]
3855 klass = commands[cmdName]
3858 print "unknown command %s" % cmdName
3860 printUsage(commands.keys())
3863 options = cmd.options
3864 cmd.gitdir = os.environ.get("GIT_DIR", None)
3868 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3870 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3872 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3874 description = cmd.description,
3875 formatter = HelpFormatter())
3877 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3879 verbose = cmd.verbose
3881 if cmd.gitdir == None:
3882 cmd.gitdir = os.path.abspath(".git")
3883 if not isValidGitDir(cmd.gitdir):
3884 # "rev-parse --git-dir" without arguments will try $PWD/.git
3885 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3886 if os.path.exists(cmd.gitdir):
3887 cdup = read_pipe("git rev-parse --show-cdup").strip()
3891 if not isValidGitDir(cmd.gitdir):
3892 if isValidGitDir(cmd.gitdir + "/.git"):
3893 cmd.gitdir += "/.git"
3895 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3897 # so git commands invoked from the P4 workspace will succeed
3898 os.environ["GIT_DIR"] = cmd.gitdir
3900 if not cmd.run(args):
3905 if __name__ == '__main__':