3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self, returncode, cmd):
39 self.returncode = returncode
42 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
52 def p4_build_cmd(cmd):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user = gitConfig("git-p4.user")
63 real_cmd += ["-u",user]
65 password = gitConfig("git-p4.password")
67 real_cmd += ["-P", password]
69 port = gitConfig("git-p4.port")
71 real_cmd += ["-p", port]
73 host = gitConfig("git-p4.host")
75 real_cmd += ["-H", host]
77 client = gitConfig("git-p4.client")
79 real_cmd += ["-c", client]
81 retries = gitConfigInt("git-p4.retries")
83 # Perform 3 retries by default
85 real_cmd += ["-r", str(retries)]
87 if isinstance(cmd,basestring):
88 real_cmd = ' '.join(real_cmd) + ' ' + cmd
93 def chdir(path, is_client_path=False):
94 """Do chdir to the given path, and set the PWD environment
95 variable for use by P4. It does not look at getcwd() output.
96 Since we're not using the shell, it is necessary to set the
97 PWD environment variable explicitly.
99 Normally, expand the path to force it to be absolute. This
100 addresses the use of relative path names inside P4 settings,
101 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
102 as given; it looks for .p4config using PWD.
104 If is_client_path, the path was handed to us directly by p4,
105 and may be a symbolic link. Do not call os.getcwd() in this
106 case, because it will cause p4 to think that PWD is not inside
111 if not is_client_path:
113 os.environ['PWD'] = path
116 """Return free space in bytes on the disk of the given dirname."""
117 if platform.system() == 'Windows':
118 free_bytes = ctypes.c_ulonglong(0)
119 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
120 return free_bytes.value
122 st = os.statvfs(os.getcwd())
123 return st.f_bavail * st.f_frsize
129 sys.stderr.write(msg + "\n")
132 def write_pipe(c, stdin):
134 sys.stderr.write('Writing pipe: %s\n' % str(c))
136 expand = isinstance(c,basestring)
137 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
139 val = pipe.write(stdin)
142 die('Command failed: %s' % str(c))
146 def p4_write_pipe(c, stdin):
147 real_cmd = p4_build_cmd(c)
148 return write_pipe(real_cmd, stdin)
150 def read_pipe(c, ignore_error=False):
152 sys.stderr.write('Reading pipe: %s\n' % str(c))
154 expand = isinstance(c,basestring)
155 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
156 (out, err) = p.communicate()
157 if p.returncode != 0 and not ignore_error:
158 die('Command failed: %s\nError: %s' % (str(c), err))
161 def p4_read_pipe(c, ignore_error=False):
162 real_cmd = p4_build_cmd(c)
163 return read_pipe(real_cmd, ignore_error)
165 def read_pipe_lines(c):
167 sys.stderr.write('Reading pipe: %s\n' % str(c))
169 expand = isinstance(c, basestring)
170 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
172 val = pipe.readlines()
173 if pipe.close() or p.wait():
174 die('Command failed: %s' % str(c))
178 def p4_read_pipe_lines(c):
179 """Specifically invoke p4 on the command supplied. """
180 real_cmd = p4_build_cmd(c)
181 return read_pipe_lines(real_cmd)
183 def p4_has_command(cmd):
184 """Ask p4 for help on this command. If it returns an error, the
185 command does not exist in this version of p4."""
186 real_cmd = p4_build_cmd(["help", cmd])
187 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
188 stderr=subprocess.PIPE)
190 return p.returncode == 0
192 def p4_has_move_command():
193 """See if the move command exists, that it supports -k, and that
194 it has not been administratively disabled. The arguments
195 must be correct, but the filenames do not have to exist. Use
196 ones with wildcards so even if they exist, it will fail."""
198 if not p4_has_command("move"):
200 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
201 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
202 (out, err) = p.communicate()
203 # return code will be 1 in either case
204 if err.find("Invalid option") >= 0:
206 if err.find("disabled") >= 0:
208 # assume it failed because @... was invalid changelist
211 def system(cmd, ignore_error=False):
212 expand = isinstance(cmd,basestring)
214 sys.stderr.write("executing %s\n" % str(cmd))
215 retcode = subprocess.call(cmd, shell=expand)
216 if retcode and not ignore_error:
217 raise CalledProcessError(retcode, cmd)
222 """Specifically invoke p4 as the system command. """
223 real_cmd = p4_build_cmd(cmd)
224 expand = isinstance(real_cmd, basestring)
225 retcode = subprocess.call(real_cmd, shell=expand)
227 raise CalledProcessError(retcode, real_cmd)
229 _p4_version_string = None
230 def p4_version_string():
231 """Read the version string, showing just the last line, which
232 hopefully is the interesting version bit.
235 Perforce - The Fast Software Configuration Management System.
236 Copyright 1995-2011 Perforce Software. All rights reserved.
237 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
239 global _p4_version_string
240 if not _p4_version_string:
241 a = p4_read_pipe_lines(["-V"])
242 _p4_version_string = a[-1].rstrip()
243 return _p4_version_string
245 def p4_integrate(src, dest):
246 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
248 def p4_sync(f, *options):
249 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
252 # forcibly add file names with wildcards
253 if wildcard_present(f):
254 p4_system(["add", "-f", f])
256 p4_system(["add", f])
259 p4_system(["delete", wildcard_encode(f)])
261 def p4_edit(f, *options):
262 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
265 p4_system(["revert", wildcard_encode(f)])
267 def p4_reopen(type, f):
268 p4_system(["reopen", "-t", type, wildcard_encode(f)])
270 def p4_reopen_in_change(changelist, files):
271 cmd = ["reopen", "-c", str(changelist)] + files
274 def p4_move(src, dest):
275 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
277 def p4_last_change():
278 results = p4CmdList(["changes", "-m", "1"])
279 return int(results[0]['change'])
281 def p4_describe(change):
282 """Make sure it returns a valid result by checking for
283 the presence of field "time". Return a dict of the
286 ds = p4CmdList(["describe", "-s", str(change)])
288 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
292 if "p4ExitCode" in d:
293 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
296 if d["code"] == "error":
297 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
300 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
305 # Canonicalize the p4 type and return a tuple of the
306 # base type, plus any modifiers. See "p4 help filetypes"
307 # for a list and explanation.
309 def split_p4_type(p4type):
311 p4_filetypes_historical = {
312 "ctempobj": "binary+Sw",
318 "tempobj": "binary+FSw",
319 "ubinary": "binary+F",
320 "uresource": "resource+F",
321 "uxbinary": "binary+Fx",
322 "xbinary": "binary+x",
324 "xtempobj": "binary+Swx",
326 "xunicode": "unicode+x",
329 if p4type in p4_filetypes_historical:
330 p4type = p4_filetypes_historical[p4type]
332 s = p4type.split("+")
340 # return the raw p4 type of a file (text, text+ko, etc)
343 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
344 return results[0]['headType']
347 # Given a type base and modifier, return a regexp matching
348 # the keywords that can be expanded in the file
350 def p4_keywords_regexp_for_type(base, type_mods):
351 if base in ("text", "unicode", "binary"):
353 if "ko" in type_mods:
355 elif "k" in type_mods:
356 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
360 \$ # Starts with a dollar, followed by...
361 (%s) # one of the keywords, followed by...
362 (:[^$\n]+)? # possibly an old expansion, followed by...
370 # Given a file, return a regexp matching the possible
371 # RCS keywords that will be expanded, or None for files
372 # with kw expansion turned off.
374 def p4_keywords_regexp_for_file(file):
375 if not os.path.exists(file):
378 (type_base, type_mods) = split_p4_type(p4_type(file))
379 return p4_keywords_regexp_for_type(type_base, type_mods)
381 def setP4ExecBit(file, mode):
382 # Reopens an already open file and changes the execute bit to match
383 # the execute bit setting in the passed in mode.
387 if not isModeExec(mode):
388 p4Type = getP4OpenedType(file)
389 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
390 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
391 if p4Type[-1] == "+":
392 p4Type = p4Type[0:-1]
394 p4_reopen(p4Type, file)
396 def getP4OpenedType(file):
397 # Returns the perforce file type for the given file.
399 result = p4_read_pipe(["opened", wildcard_encode(file)])
400 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
402 return match.group(1)
404 die("Could not determine file type for %s (result: '%s')" % (file, result))
406 # Return the set of all p4 labels
407 def getP4Labels(depotPaths):
409 if isinstance(depotPaths,basestring):
410 depotPaths = [depotPaths]
412 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
418 # Return the set of all git tags
421 for line in read_pipe_lines(["git", "tag"]):
426 def diffTreePattern():
427 # This is a simple generator for the diff tree regex pattern. This could be
428 # a class variable if this and parseDiffTreeEntry were a part of a class.
429 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
433 def parseDiffTreeEntry(entry):
434 """Parses a single diff tree entry into its component elements.
436 See git-diff-tree(1) manpage for details about the format of the diff
437 output. This method returns a dictionary with the following elements:
439 src_mode - The mode of the source file
440 dst_mode - The mode of the destination file
441 src_sha1 - The sha1 for the source file
442 dst_sha1 - The sha1 fr the destination file
443 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
444 status_score - The score for the status (applicable for 'C' and 'R'
445 statuses). This is None if there is no score.
446 src - The path for the source file.
447 dst - The path for the destination file. This is only present for
448 copy or renames. If it is not present, this is None.
450 If the pattern is not matched, None is returned."""
452 match = diffTreePattern().next().match(entry)
455 'src_mode': match.group(1),
456 'dst_mode': match.group(2),
457 'src_sha1': match.group(3),
458 'dst_sha1': match.group(4),
459 'status': match.group(5),
460 'status_score': match.group(6),
461 'src': match.group(7),
462 'dst': match.group(10)
466 def isModeExec(mode):
467 # Returns True if the given git mode represents an executable file,
469 return mode[-3:] == "755"
471 def isModeExecChanged(src_mode, dst_mode):
472 return isModeExec(src_mode) != isModeExec(dst_mode)
474 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
476 if isinstance(cmd,basestring):
483 cmd = p4_build_cmd(cmd)
485 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
487 # Use a temporary file to avoid deadlocks without
488 # subprocess.communicate(), which would put another copy
489 # of stdout into memory.
491 if stdin is not None:
492 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
493 if isinstance(stdin,basestring):
494 stdin_file.write(stdin)
497 stdin_file.write(i + '\n')
501 p4 = subprocess.Popen(cmd,
504 stdout=subprocess.PIPE)
509 entry = marshal.load(p4.stdout)
519 entry["p4ExitCode"] = exitCode
525 list = p4CmdList(cmd)
531 def p4Where(depotPath):
532 if not depotPath.endswith("/"):
534 depotPathLong = depotPath + "..."
535 outputList = p4CmdList(["where", depotPathLong])
537 for entry in outputList:
538 if "depotFile" in entry:
539 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
540 # The base path always ends with "/...".
541 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
544 elif "data" in entry:
545 data = entry.get("data")
546 space = data.find(" ")
547 if data[:space] == depotPath:
552 if output["code"] == "error":
556 clientPath = output.get("path")
557 elif "data" in output:
558 data = output.get("data")
559 lastSpace = data.rfind(" ")
560 clientPath = data[lastSpace + 1:]
562 if clientPath.endswith("..."):
563 clientPath = clientPath[:-3]
566 def currentGitBranch():
567 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
572 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
574 def isValidGitDir(path):
575 if (os.path.exists(path + "/HEAD")
576 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
580 def parseRevision(ref):
581 return read_pipe("git rev-parse %s" % ref).strip()
583 def branchExists(ref):
584 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
588 def extractLogMessageFromGitCommit(commit):
591 ## fixme: title is first line of commit, not 1st paragraph.
593 for log in read_pipe_lines("git cat-file commit %s" % commit):
602 def extractSettingsGitLog(log):
604 for line in log.split("\n"):
606 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
610 assignments = m.group(1).split (':')
611 for a in assignments:
613 key = vals[0].strip()
614 val = ('='.join (vals[1:])).strip()
615 if val.endswith ('\"') and val.startswith('"'):
620 paths = values.get("depot-paths")
622 paths = values.get("depot-path")
624 values['depot-paths'] = paths.split(',')
627 def gitBranchExists(branch):
628 proc = subprocess.Popen(["git", "rev-parse", branch],
629 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
630 return proc.wait() == 0;
634 def gitConfig(key, typeSpecifier=None):
635 if not _gitConfig.has_key(key):
636 cmd = [ "git", "config" ]
638 cmd += [ typeSpecifier ]
640 s = read_pipe(cmd, ignore_error=True)
641 _gitConfig[key] = s.strip()
642 return _gitConfig[key]
644 def gitConfigBool(key):
645 """Return a bool, using git config --bool. It is True only if the
646 variable is set to true, and False if set to false or not present
649 if not _gitConfig.has_key(key):
650 _gitConfig[key] = gitConfig(key, '--bool') == "true"
651 return _gitConfig[key]
653 def gitConfigInt(key):
654 if not _gitConfig.has_key(key):
655 cmd = [ "git", "config", "--int", key ]
656 s = read_pipe(cmd, ignore_error=True)
659 _gitConfig[key] = int(gitConfig(key, '--int'))
661 _gitConfig[key] = None
662 return _gitConfig[key]
664 def gitConfigList(key):
665 if not _gitConfig.has_key(key):
666 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
667 _gitConfig[key] = s.strip().split(os.linesep)
668 if _gitConfig[key] == ['']:
670 return _gitConfig[key]
672 def p4BranchesInGit(branchesAreInRemotes=True):
673 """Find all the branches whose names start with "p4/", looking
674 in remotes or heads as specified by the argument. Return
675 a dictionary of { branch: revision } for each one found.
676 The branch names are the short names, without any
681 cmdline = "git rev-parse --symbolic "
682 if branchesAreInRemotes:
683 cmdline += "--remotes"
685 cmdline += "--branches"
687 for line in read_pipe_lines(cmdline):
691 if not line.startswith('p4/'):
693 # special symbolic ref to p4/master
694 if line == "p4/HEAD":
697 # strip off p4/ prefix
698 branch = line[len("p4/"):]
700 branches[branch] = parseRevision(line)
704 def branch_exists(branch):
705 """Make sure that the given ref name really exists."""
707 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
708 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
709 out, _ = p.communicate()
712 # expect exactly one line of output: the branch name
713 return out.rstrip() == branch
715 def findUpstreamBranchPoint(head = "HEAD"):
716 branches = p4BranchesInGit()
717 # map from depot-path to branch name
718 branchByDepotPath = {}
719 for branch in branches.keys():
720 tip = branches[branch]
721 log = extractLogMessageFromGitCommit(tip)
722 settings = extractSettingsGitLog(log)
723 if settings.has_key("depot-paths"):
724 paths = ",".join(settings["depot-paths"])
725 branchByDepotPath[paths] = "remotes/p4/" + branch
729 while parent < 65535:
730 commit = head + "~%s" % parent
731 log = extractLogMessageFromGitCommit(commit)
732 settings = extractSettingsGitLog(log)
733 if settings.has_key("depot-paths"):
734 paths = ",".join(settings["depot-paths"])
735 if branchByDepotPath.has_key(paths):
736 return [branchByDepotPath[paths], settings]
740 return ["", settings]
742 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
744 print ("Creating/updating branch(es) in %s based on origin branch(es)"
747 originPrefix = "origin/p4/"
749 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
751 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
754 headName = line[len(originPrefix):]
755 remoteHead = localRefPrefix + headName
758 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
759 if (not original.has_key('depot-paths')
760 or not original.has_key('change')):
764 if not gitBranchExists(remoteHead):
766 print "creating %s" % remoteHead
769 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
770 if settings.has_key('change') > 0:
771 if settings['depot-paths'] == original['depot-paths']:
772 originP4Change = int(original['change'])
773 p4Change = int(settings['change'])
774 if originP4Change > p4Change:
775 print ("%s (%s) is newer than %s (%s). "
776 "Updating p4 branch from origin."
777 % (originHead, originP4Change,
778 remoteHead, p4Change))
781 print ("Ignoring: %s was imported from %s while "
782 "%s was imported from %s"
783 % (originHead, ','.join(original['depot-paths']),
784 remoteHead, ','.join(settings['depot-paths'])))
787 system("git update-ref %s %s" % (remoteHead, originHead))
789 def originP4BranchesExist():
790 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
793 def p4ParseNumericChangeRange(parts):
794 changeStart = int(parts[0][1:])
795 if parts[1] == '#head':
796 changeEnd = p4_last_change()
798 changeEnd = int(parts[1])
800 return (changeStart, changeEnd)
802 def chooseBlockSize(blockSize):
806 return defaultBlockSize
808 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
811 # Parse the change range into start and end. Try to find integer
812 # revision ranges as these can be broken up into blocks to avoid
813 # hitting server-side limits (maxrows, maxscanresults). But if
814 # that doesn't work, fall back to using the raw revision specifier
815 # strings, without using block mode.
817 if changeRange is None or changeRange == '':
819 changeEnd = p4_last_change()
820 block_size = chooseBlockSize(requestedBlockSize)
822 parts = changeRange.split(',')
823 assert len(parts) == 2
825 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
826 block_size = chooseBlockSize(requestedBlockSize)
828 changeStart = parts[0][1:]
830 if requestedBlockSize:
831 die("cannot use --changes-block-size with non-numeric revisions")
836 # Retrieve changes a block at a time, to prevent running
837 # into a MaxResults/MaxScanRows error from the server.
843 end = min(changeEnd, changeStart + block_size)
844 revisionRange = "%d,%d" % (changeStart, end)
846 revisionRange = "%s,%s" % (changeStart, changeEnd)
849 cmd += ["%s...@%s" % (p, revisionRange)]
851 # Insert changes in chronological order
852 for line in reversed(p4_read_pipe_lines(cmd)):
853 changes.append(int(line.split(" ")[1]))
861 changeStart = end + 1
863 changes = sorted(changes)
866 def p4PathStartsWith(path, prefix):
867 # This method tries to remedy a potential mixed-case issue:
869 # If UserA adds //depot/DirA/file1
870 # and UserB adds //depot/dira/file2
872 # we may or may not have a problem. If you have core.ignorecase=true,
873 # we treat DirA and dira as the same directory
874 if gitConfigBool("core.ignorecase"):
875 return path.lower().startswith(prefix.lower())
876 return path.startswith(prefix)
879 """Look at the p4 client spec, create a View() object that contains
880 all the mappings, and return it."""
882 specList = p4CmdList("client -o")
883 if len(specList) != 1:
884 die('Output from "client -o" is %d lines, expecting 1' %
887 # dictionary of all client parameters
891 client_name = entry["Client"]
893 # just the keys that start with "View"
894 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
897 view = View(client_name)
899 # append the lines, in order, to the view
900 for view_num in range(len(view_keys)):
901 k = "View%d" % view_num
902 if k not in view_keys:
903 die("Expected view key %s missing" % k)
904 view.append(entry[k])
909 """Grab the client directory."""
911 output = p4CmdList("client -o")
913 die('Output from "client -o" is %d lines, expecting 1' % len(output))
916 if "Root" not in entry:
917 die('Client has no "Root"')
922 # P4 wildcards are not allowed in filenames. P4 complains
923 # if you simply add them, but you can force it with "-f", in
924 # which case it translates them into %xx encoding internally.
926 def wildcard_decode(path):
927 # Search for and fix just these four characters. Do % last so
928 # that fixing it does not inadvertently create new %-escapes.
929 # Cannot have * in a filename in windows; untested as to
930 # what p4 would do in such a case.
931 if not platform.system() == "Windows":
932 path = path.replace("%2A", "*")
933 path = path.replace("%23", "#") \
934 .replace("%40", "@") \
938 def wildcard_encode(path):
939 # do % first to avoid double-encoding the %s introduced here
940 path = path.replace("%", "%25") \
941 .replace("*", "%2A") \
942 .replace("#", "%23") \
946 def wildcard_present(path):
947 m = re.search("[*#@%]", path)
950 class LargeFileSystem(object):
951 """Base class for large file system support."""
953 def __init__(self, writeToGitStream):
954 self.largeFiles = set()
955 self.writeToGitStream = writeToGitStream
957 def generatePointer(self, cloneDestination, contentFile):
958 """Return the content of a pointer file that is stored in Git instead of
959 the actual content."""
960 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
962 def pushFile(self, localLargeFile):
963 """Push the actual content which is not stored in the Git repository to
965 assert False, "Method 'pushFile' required in " + self.__class__.__name__
967 def hasLargeFileExtension(self, relPath):
970 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
974 def generateTempFile(self, contents):
975 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
979 return contentFile.name
981 def exceedsLargeFileThreshold(self, relPath, contents):
982 if gitConfigInt('git-p4.largeFileThreshold'):
983 contentsSize = sum(len(d) for d in contents)
984 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
986 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
987 contentsSize = sum(len(d) for d in contents)
988 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
990 contentTempFile = self.generateTempFile(contents)
991 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
992 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
993 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
995 compressedContentsSize = zf.infolist()[0].compress_size
996 os.remove(contentTempFile)
997 os.remove(compressedContentFile.name)
998 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1002 def addLargeFile(self, relPath):
1003 self.largeFiles.add(relPath)
1005 def removeLargeFile(self, relPath):
1006 self.largeFiles.remove(relPath)
1008 def isLargeFile(self, relPath):
1009 return relPath in self.largeFiles
1011 def processContent(self, git_mode, relPath, contents):
1012 """Processes the content of git fast import. This method decides if a
1013 file is stored in the large file system and handles all necessary
1015 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1016 contentTempFile = self.generateTempFile(contents)
1017 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1018 if pointer_git_mode:
1019 git_mode = pointer_git_mode
1021 # Move temp file to final location in large file system
1022 largeFileDir = os.path.dirname(localLargeFile)
1023 if not os.path.isdir(largeFileDir):
1024 os.makedirs(largeFileDir)
1025 shutil.move(contentTempFile, localLargeFile)
1026 self.addLargeFile(relPath)
1027 if gitConfigBool('git-p4.largeFilePush'):
1028 self.pushFile(localLargeFile)
1030 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1031 return (git_mode, contents)
1033 class MockLFS(LargeFileSystem):
1034 """Mock large file system for testing."""
1036 def generatePointer(self, contentFile):
1037 """The pointer content is the original content prefixed with "pointer-".
1038 The local filename of the large file storage is derived from the file content.
1040 with open(contentFile, 'r') as f:
1043 pointerContents = 'pointer-' + content
1044 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1045 return (gitMode, pointerContents, localLargeFile)
1047 def pushFile(self, localLargeFile):
1048 """The remote filename of the large file storage is the same as the local
1049 one but in a different directory.
1051 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1052 if not os.path.exists(remotePath):
1053 os.makedirs(remotePath)
1054 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1056 class GitLFS(LargeFileSystem):
1057 """Git LFS as backend for the git-p4 large file system.
1058 See https://git-lfs.github.com/ for details."""
1060 def __init__(self, *args):
1061 LargeFileSystem.__init__(self, *args)
1062 self.baseGitAttributes = []
1064 def generatePointer(self, contentFile):
1065 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1066 mode and content which is stored in the Git repository instead of
1067 the actual content. Return also the new location of the actual
1070 if os.path.getsize(contentFile) == 0:
1071 return (None, '', None)
1073 pointerProcess = subprocess.Popen(
1074 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1075 stdout=subprocess.PIPE
1077 pointerFile = pointerProcess.stdout.read()
1078 if pointerProcess.wait():
1079 os.remove(contentFile)
1080 die('git-lfs pointer command failed. Did you install the extension?')
1082 # Git LFS removed the preamble in the output of the 'pointer' command
1083 # starting from version 1.2.0. Check for the preamble here to support
1085 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1086 if pointerFile.startswith('Git LFS pointer for'):
1087 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1089 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1090 localLargeFile = os.path.join(
1092 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1095 # LFS Spec states that pointer files should not have the executable bit set.
1097 return (gitMode, pointerFile, localLargeFile)
1099 def pushFile(self, localLargeFile):
1100 uploadProcess = subprocess.Popen(
1101 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1103 if uploadProcess.wait():
1104 die('git-lfs push command failed. Did you define a remote?')
1106 def generateGitAttributes(self):
1108 self.baseGitAttributes +
1112 '# Git LFS (see https://git-lfs.github.com/)\n',
1115 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1116 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1118 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1119 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1123 def addLargeFile(self, relPath):
1124 LargeFileSystem.addLargeFile(self, relPath)
1125 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1127 def removeLargeFile(self, relPath):
1128 LargeFileSystem.removeLargeFile(self, relPath)
1129 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1131 def processContent(self, git_mode, relPath, contents):
1132 if relPath == '.gitattributes':
1133 self.baseGitAttributes = contents
1134 return (git_mode, self.generateGitAttributes())
1136 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1140 self.usage = "usage: %prog [options]"
1141 self.needsGit = True
1142 self.verbose = False
1146 self.userMapFromPerforceServer = False
1147 self.myP4UserId = None
1151 return self.myP4UserId
1153 results = p4CmdList("user -o")
1155 if r.has_key('User'):
1156 self.myP4UserId = r['User']
1158 die("Could not find your p4 user id")
1160 def p4UserIsMe(self, p4User):
1161 # return True if the given p4 user is actually me
1162 me = self.p4UserId()
1163 if not p4User or p4User != me:
1168 def getUserCacheFilename(self):
1169 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1170 return home + "/.gitp4-usercache.txt"
1172 def getUserMapFromPerforceServer(self):
1173 if self.userMapFromPerforceServer:
1178 for output in p4CmdList("users"):
1179 if not output.has_key("User"):
1181 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1182 self.emails[output["Email"]] = output["User"]
1184 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1185 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1186 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1187 if mapUser and len(mapUser[0]) == 3:
1188 user = mapUser[0][0]
1189 fullname = mapUser[0][1]
1190 email = mapUser[0][2]
1191 self.users[user] = fullname + " <" + email + ">"
1192 self.emails[email] = user
1195 for (key, val) in self.users.items():
1196 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1198 open(self.getUserCacheFilename(), "wb").write(s)
1199 self.userMapFromPerforceServer = True
1201 def loadUserMapFromCache(self):
1203 self.userMapFromPerforceServer = False
1205 cache = open(self.getUserCacheFilename(), "rb")
1206 lines = cache.readlines()
1209 entry = line.strip().split("\t")
1210 self.users[entry[0]] = entry[1]
1212 self.getUserMapFromPerforceServer()
1214 class P4Debug(Command):
1216 Command.__init__(self)
1218 self.description = "A tool to debug the output of p4 -G."
1219 self.needsGit = False
1221 def run(self, args):
1223 for output in p4CmdList(args):
1224 print 'Element: %d' % j
1229 class P4RollBack(Command):
1231 Command.__init__(self)
1233 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1235 self.description = "A tool to debug the multi-branch import. Don't use :)"
1236 self.rollbackLocalBranches = False
1238 def run(self, args):
1241 maxChange = int(args[0])
1243 if "p4ExitCode" in p4Cmd("changes -m 1"):
1244 die("Problems executing p4");
1246 if self.rollbackLocalBranches:
1247 refPrefix = "refs/heads/"
1248 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1250 refPrefix = "refs/remotes/"
1251 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1254 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1256 ref = refPrefix + line
1257 log = extractLogMessageFromGitCommit(ref)
1258 settings = extractSettingsGitLog(log)
1260 depotPaths = settings['depot-paths']
1261 change = settings['change']
1265 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1266 for p in depotPaths]))) == 0:
1267 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1268 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1271 while change and int(change) > maxChange:
1274 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1275 system("git update-ref %s \"%s^\"" % (ref, ref))
1276 log = extractLogMessageFromGitCommit(ref)
1277 settings = extractSettingsGitLog(log)
1280 depotPaths = settings['depot-paths']
1281 change = settings['change']
1284 print "%s rewound to %s" % (ref, change)
1288 class P4Submit(Command, P4UserMap):
1290 conflict_behavior_choices = ("ask", "skip", "quit")
1293 Command.__init__(self)
1294 P4UserMap.__init__(self)
1296 optparse.make_option("--origin", dest="origin"),
1297 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1298 # preserve the user, requires relevant p4 permissions
1299 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1300 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1301 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1302 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1303 optparse.make_option("--conflict", dest="conflict_behavior",
1304 choices=self.conflict_behavior_choices),
1305 optparse.make_option("--branch", dest="branch"),
1306 optparse.make_option("--shelve", dest="shelve", action="store_true",
1307 help="Shelve instead of submit. Shelved files are reverted, "
1308 "restoring the workspace to the state before the shelve"),
1309 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1310 metavar="CHANGELIST",
1311 help="update an existing shelved changelist, implies --shelve")
1313 self.description = "Submit changes from git to the perforce depot."
1314 self.usage += " [name of git branch to submit into perforce depot]"
1316 self.detectRenames = False
1317 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1318 self.dry_run = False
1320 self.update_shelve = None
1321 self.prepare_p4_only = False
1322 self.conflict_behavior = None
1323 self.isWindows = (platform.system() == "Windows")
1324 self.exportLabels = False
1325 self.p4HasMoveCommand = p4_has_move_command()
1328 if gitConfig('git-p4.largeFileSystem'):
1329 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1332 if len(p4CmdList("opened ...")) > 0:
1333 die("You have files opened with perforce! Close them before starting the sync.")
1335 def separate_jobs_from_description(self, message):
1336 """Extract and return a possible Jobs field in the commit
1337 message. It goes into a separate section in the p4 change
1340 A jobs line starts with "Jobs:" and looks like a new field
1341 in a form. Values are white-space separated on the same
1342 line or on following lines that start with a tab.
1344 This does not parse and extract the full git commit message
1345 like a p4 form. It just sees the Jobs: line as a marker
1346 to pass everything from then on directly into the p4 form,
1347 but outside the description section.
1349 Return a tuple (stripped log message, jobs string)."""
1351 m = re.search(r'^Jobs:', message, re.MULTILINE)
1353 return (message, None)
1355 jobtext = message[m.start():]
1356 stripped_message = message[:m.start()].rstrip()
1357 return (stripped_message, jobtext)
1359 def prepareLogMessage(self, template, message, jobs):
1360 """Edits the template returned from "p4 change -o" to insert
1361 the message in the Description field, and the jobs text in
1365 inDescriptionSection = False
1367 for line in template.split("\n"):
1368 if line.startswith("#"):
1369 result += line + "\n"
1372 if inDescriptionSection:
1373 if line.startswith("Files:") or line.startswith("Jobs:"):
1374 inDescriptionSection = False
1375 # insert Jobs section
1377 result += jobs + "\n"
1381 if line.startswith("Description:"):
1382 inDescriptionSection = True
1384 for messageLine in message.split("\n"):
1385 line += "\t" + messageLine + "\n"
1387 result += line + "\n"
1391 def patchRCSKeywords(self, file, pattern):
1392 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1393 (handle, outFileName) = tempfile.mkstemp(dir='.')
1395 outFile = os.fdopen(handle, "w+")
1396 inFile = open(file, "r")
1397 regexp = re.compile(pattern, re.VERBOSE)
1398 for line in inFile.readlines():
1399 line = regexp.sub(r'$\1$', line)
1403 # Forcibly overwrite the original file
1405 shutil.move(outFileName, file)
1407 # cleanup our temporary file
1408 os.unlink(outFileName)
1409 print "Failed to strip RCS keywords in %s" % file
1412 print "Patched up RCS keywords in %s" % file
1414 def p4UserForCommit(self,id):
1415 # Return the tuple (perforce user,git email) for a given git commit id
1416 self.getUserMapFromPerforceServer()
1417 gitEmail = read_pipe(["git", "log", "--max-count=1",
1418 "--format=%ae", id])
1419 gitEmail = gitEmail.strip()
1420 if not self.emails.has_key(gitEmail):
1421 return (None,gitEmail)
1423 return (self.emails[gitEmail],gitEmail)
1425 def checkValidP4Users(self,commits):
1426 # check if any git authors cannot be mapped to p4 users
1428 (user,email) = self.p4UserForCommit(id)
1430 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1431 if gitConfigBool("git-p4.allowMissingP4Users"):
1434 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1436 def lastP4Changelist(self):
1437 # Get back the last changelist number submitted in this client spec. This
1438 # then gets used to patch up the username in the change. If the same
1439 # client spec is being used by multiple processes then this might go
1441 results = p4CmdList("client -o") # find the current client
1444 if r.has_key('Client'):
1445 client = r['Client']
1448 die("could not get client spec")
1449 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1451 if r.has_key('change'):
1453 die("Could not get changelist number for last submit - cannot patch up user details")
1455 def modifyChangelistUser(self, changelist, newUser):
1456 # fixup the user field of a changelist after it has been submitted.
1457 changes = p4CmdList("change -o %s" % changelist)
1458 if len(changes) != 1:
1459 die("Bad output from p4 change modifying %s to user %s" %
1460 (changelist, newUser))
1463 if c['User'] == newUser: return # nothing to do
1465 input = marshal.dumps(c)
1467 result = p4CmdList("change -f -i", stdin=input)
1469 if r.has_key('code'):
1470 if r['code'] == 'error':
1471 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1472 if r.has_key('data'):
1473 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1475 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1477 def canChangeChangelists(self):
1478 # check to see if we have p4 admin or super-user permissions, either of
1479 # which are required to modify changelists.
1480 results = p4CmdList(["protects", self.depotPath])
1482 if r.has_key('perm'):
1483 if r['perm'] == 'admin':
1485 if r['perm'] == 'super':
1489 def prepareSubmitTemplate(self, changelist=None):
1490 """Run "p4 change -o" to grab a change specification template.
1491 This does not use "p4 -G", as it is nice to keep the submission
1492 template in original order, since a human might edit it.
1494 Remove lines in the Files section that show changes to files
1495 outside the depot path we're committing into."""
1497 [upstream, settings] = findUpstreamBranchPoint()
1500 inFilesSection = False
1501 args = ['change', '-o']
1503 args.append(str(changelist))
1505 for line in p4_read_pipe_lines(args):
1506 if line.endswith("\r\n"):
1507 line = line[:-2] + "\n"
1509 if line.startswith("\t"):
1510 # path starts and ends with a tab
1512 lastTab = path.rfind("\t")
1514 path = path[:lastTab]
1515 if settings.has_key('depot-paths'):
1516 if not [p for p in settings['depot-paths']
1517 if p4PathStartsWith(path, p)]:
1520 if not p4PathStartsWith(path, self.depotPath):
1523 inFilesSection = False
1525 if line.startswith("Files:"):
1526 inFilesSection = True
1532 def edit_template(self, template_file):
1533 """Invoke the editor to let the user change the submission
1534 message. Return true if okay to continue with the submit."""
1536 # if configured to skip the editing part, just submit
1537 if gitConfigBool("git-p4.skipSubmitEdit"):
1540 # look at the modification time, to check later if the user saved
1542 mtime = os.stat(template_file).st_mtime
1545 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1546 editor = os.environ.get("P4EDITOR")
1548 editor = read_pipe("git var GIT_EDITOR").strip()
1549 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1551 # If the file was not saved, prompt to see if this patch should
1552 # be skipped. But skip this verification step if configured so.
1553 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1556 # modification time updated means user saved the file
1557 if os.stat(template_file).st_mtime > mtime:
1561 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1567 def get_diff_description(self, editedFiles, filesToAdd):
1569 if os.environ.has_key("P4DIFF"):
1570 del(os.environ["P4DIFF"])
1572 for editedFile in editedFiles:
1573 diff += p4_read_pipe(['diff', '-du',
1574 wildcard_encode(editedFile)])
1578 for newFile in filesToAdd:
1579 newdiff += "==== new file ====\n"
1580 newdiff += "--- /dev/null\n"
1581 newdiff += "+++ %s\n" % newFile
1582 f = open(newFile, "r")
1583 for line in f.readlines():
1584 newdiff += "+" + line
1587 return (diff + newdiff).replace('\r\n', '\n')
1589 def applyCommit(self, id):
1590 """Apply one commit, return True if it succeeded."""
1592 print "Applying", read_pipe(["git", "show", "-s",
1593 "--format=format:%h %s", id])
1595 (p4User, gitEmail) = self.p4UserForCommit(id)
1597 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1599 filesToChangeType = set()
1600 filesToDelete = set()
1602 pureRenameCopy = set()
1603 filesToChangeExecBit = {}
1607 diff = parseDiffTreeEntry(line)
1608 modifier = diff['status']
1610 all_files.append(path)
1614 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1615 filesToChangeExecBit[path] = diff['dst_mode']
1616 editedFiles.add(path)
1617 elif modifier == "A":
1618 filesToAdd.add(path)
1619 filesToChangeExecBit[path] = diff['dst_mode']
1620 if path in filesToDelete:
1621 filesToDelete.remove(path)
1622 elif modifier == "D":
1623 filesToDelete.add(path)
1624 if path in filesToAdd:
1625 filesToAdd.remove(path)
1626 elif modifier == "C":
1627 src, dest = diff['src'], diff['dst']
1628 p4_integrate(src, dest)
1629 pureRenameCopy.add(dest)
1630 if diff['src_sha1'] != diff['dst_sha1']:
1632 pureRenameCopy.discard(dest)
1633 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1635 pureRenameCopy.discard(dest)
1636 filesToChangeExecBit[dest] = diff['dst_mode']
1638 # turn off read-only attribute
1639 os.chmod(dest, stat.S_IWRITE)
1641 editedFiles.add(dest)
1642 elif modifier == "R":
1643 src, dest = diff['src'], diff['dst']
1644 if self.p4HasMoveCommand:
1645 p4_edit(src) # src must be open before move
1646 p4_move(src, dest) # opens for (move/delete, move/add)
1648 p4_integrate(src, dest)
1649 if diff['src_sha1'] != diff['dst_sha1']:
1652 pureRenameCopy.add(dest)
1653 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1654 if not self.p4HasMoveCommand:
1655 p4_edit(dest) # with move: already open, writable
1656 filesToChangeExecBit[dest] = diff['dst_mode']
1657 if not self.p4HasMoveCommand:
1659 os.chmod(dest, stat.S_IWRITE)
1661 filesToDelete.add(src)
1662 editedFiles.add(dest)
1663 elif modifier == "T":
1664 filesToChangeType.add(path)
1666 die("unknown modifier %s for %s" % (modifier, path))
1668 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1669 patchcmd = diffcmd + " | git apply "
1670 tryPatchCmd = patchcmd + "--check -"
1671 applyPatchCmd = patchcmd + "--check --apply -"
1672 patch_succeeded = True
1674 if os.system(tryPatchCmd) != 0:
1675 fixed_rcs_keywords = False
1676 patch_succeeded = False
1677 print "Unfortunately applying the change failed!"
1679 # Patch failed, maybe it's just RCS keyword woes. Look through
1680 # the patch to see if that's possible.
1681 if gitConfigBool("git-p4.attemptRCSCleanup"):
1685 for file in editedFiles | filesToDelete:
1686 # did this file's delta contain RCS keywords?
1687 pattern = p4_keywords_regexp_for_file(file)
1690 # this file is a possibility...look for RCS keywords.
1691 regexp = re.compile(pattern, re.VERBOSE)
1692 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1693 if regexp.search(line):
1695 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1696 kwfiles[file] = pattern
1699 for file in kwfiles:
1701 print "zapping %s with %s" % (line,pattern)
1702 # File is being deleted, so not open in p4. Must
1703 # disable the read-only bit on windows.
1704 if self.isWindows and file not in editedFiles:
1705 os.chmod(file, stat.S_IWRITE)
1706 self.patchRCSKeywords(file, kwfiles[file])
1707 fixed_rcs_keywords = True
1709 if fixed_rcs_keywords:
1710 print "Retrying the patch with RCS keywords cleaned up"
1711 if os.system(tryPatchCmd) == 0:
1712 patch_succeeded = True
1714 if not patch_succeeded:
1715 for f in editedFiles:
1720 # Apply the patch for real, and do add/delete/+x handling.
1722 system(applyPatchCmd)
1724 for f in filesToChangeType:
1725 p4_edit(f, "-t", "auto")
1726 for f in filesToAdd:
1728 for f in filesToDelete:
1732 # Set/clear executable bits
1733 for f in filesToChangeExecBit.keys():
1734 mode = filesToChangeExecBit[f]
1735 setP4ExecBit(f, mode)
1737 if self.update_shelve:
1738 print("all_files = %s" % str(all_files))
1739 p4_reopen_in_change(self.update_shelve, all_files)
1742 # Build p4 change description, starting with the contents
1743 # of the git commit message.
1745 logMessage = extractLogMessageFromGitCommit(id)
1746 logMessage = logMessage.strip()
1747 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1749 template = self.prepareSubmitTemplate(self.update_shelve)
1750 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1752 if self.preserveUser:
1753 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1755 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1756 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1757 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1758 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1760 separatorLine = "######## everything below this line is just the diff #######\n"
1761 if not self.prepare_p4_only:
1762 submitTemplate += separatorLine
1763 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1765 (handle, fileName) = tempfile.mkstemp()
1766 tmpFile = os.fdopen(handle, "w+b")
1768 submitTemplate = submitTemplate.replace("\n", "\r\n")
1769 tmpFile.write(submitTemplate)
1772 if self.prepare_p4_only:
1774 # Leave the p4 tree prepared, and the submit template around
1775 # and let the user decide what to do next
1778 print "P4 workspace prepared for submission."
1779 print "To submit or revert, go to client workspace"
1780 print " " + self.clientPath
1782 print "To submit, use \"p4 submit\" to write a new description,"
1783 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1784 " \"git p4\"." % fileName
1785 print "You can delete the file \"%s\" when finished." % fileName
1787 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1788 print "To preserve change ownership by user %s, you must\n" \
1789 "do \"p4 change -f <change>\" after submitting and\n" \
1790 "edit the User field."
1792 print "After submitting, renamed files must be re-synced."
1793 print "Invoke \"p4 sync -f\" on each of these files:"
1794 for f in pureRenameCopy:
1798 print "To revert the changes, use \"p4 revert ...\", and delete"
1799 print "the submit template file \"%s\"" % fileName
1801 print "Since the commit adds new files, they must be deleted:"
1802 for f in filesToAdd:
1808 # Let the user edit the change description, then submit it.
1813 if self.edit_template(fileName):
1814 # read the edited message and submit
1815 tmpFile = open(fileName, "rb")
1816 message = tmpFile.read()
1819 message = message.replace("\r\n", "\n")
1820 submitTemplate = message[:message.index(separatorLine)]
1822 if self.update_shelve:
1823 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1825 p4_write_pipe(['shelve', '-i'], submitTemplate)
1827 p4_write_pipe(['submit', '-i'], submitTemplate)
1828 # The rename/copy happened by applying a patch that created a
1829 # new file. This leaves it writable, which confuses p4.
1830 for f in pureRenameCopy:
1833 if self.preserveUser:
1835 # Get last changelist number. Cannot easily get it from
1836 # the submit command output as the output is
1838 changelist = self.lastP4Changelist()
1839 self.modifyChangelistUser(changelist, p4User)
1845 if not submitted or self.shelve:
1847 print ("Reverting shelved files.")
1849 print ("Submission cancelled, undoing p4 changes.")
1850 for f in editedFiles | filesToDelete:
1852 for f in filesToAdd:
1859 # Export git tags as p4 labels. Create a p4 label and then tag
1861 def exportGitTags(self, gitTags):
1862 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1863 if len(validLabelRegexp) == 0:
1864 validLabelRegexp = defaultLabelRegexp
1865 m = re.compile(validLabelRegexp)
1867 for name in gitTags:
1869 if not m.match(name):
1871 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1874 # Get the p4 commit this corresponds to
1875 logMessage = extractLogMessageFromGitCommit(name)
1876 values = extractSettingsGitLog(logMessage)
1878 if not values.has_key('change'):
1879 # a tag pointing to something not sent to p4; ignore
1881 print "git tag %s does not give a p4 commit" % name
1884 changelist = values['change']
1886 # Get the tag details.
1890 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1893 if re.match(r'tag\s+', l):
1895 elif re.match(r'\s*$', l):
1902 body = ["lightweight tag imported by git p4\n"]
1904 # Create the label - use the same view as the client spec we are using
1905 clientSpec = getClientSpec()
1907 labelTemplate = "Label: %s\n" % name
1908 labelTemplate += "Description:\n"
1910 labelTemplate += "\t" + b + "\n"
1911 labelTemplate += "View:\n"
1912 for depot_side in clientSpec.mappings:
1913 labelTemplate += "\t%s\n" % depot_side
1916 print "Would create p4 label %s for tag" % name
1917 elif self.prepare_p4_only:
1918 print "Not creating p4 label %s for tag due to option" \
1919 " --prepare-p4-only" % name
1921 p4_write_pipe(["label", "-i"], labelTemplate)
1924 p4_system(["tag", "-l", name] +
1925 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1928 print "created p4 label for tag %s" % name
1930 def run(self, args):
1932 self.master = currentGitBranch()
1933 elif len(args) == 1:
1934 self.master = args[0]
1935 if not branchExists(self.master):
1936 die("Branch %s does not exist" % self.master)
1941 allowSubmit = gitConfig("git-p4.allowSubmit")
1942 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1943 die("%s is not in git-p4.allowSubmit" % self.master)
1945 [upstream, settings] = findUpstreamBranchPoint()
1946 self.depotPath = settings['depot-paths'][0]
1947 if len(self.origin) == 0:
1948 self.origin = upstream
1950 if self.update_shelve:
1953 if self.preserveUser:
1954 if not self.canChangeChangelists():
1955 die("Cannot preserve user names without p4 super-user or admin permissions")
1957 # if not set from the command line, try the config file
1958 if self.conflict_behavior is None:
1959 val = gitConfig("git-p4.conflict")
1961 if val not in self.conflict_behavior_choices:
1962 die("Invalid value '%s' for config git-p4.conflict" % val)
1965 self.conflict_behavior = val
1968 print "Origin branch is " + self.origin
1970 if len(self.depotPath) == 0:
1971 print "Internal error: cannot locate perforce depot path from existing branches"
1974 self.useClientSpec = False
1975 if gitConfigBool("git-p4.useclientspec"):
1976 self.useClientSpec = True
1977 if self.useClientSpec:
1978 self.clientSpecDirs = getClientSpec()
1980 # Check for the existence of P4 branches
1981 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1983 if self.useClientSpec and not branchesDetected:
1984 # all files are relative to the client spec
1985 self.clientPath = getClientRoot()
1987 self.clientPath = p4Where(self.depotPath)
1989 if self.clientPath == "":
1990 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1992 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1993 self.oldWorkingDirectory = os.getcwd()
1995 # ensure the clientPath exists
1996 new_client_dir = False
1997 if not os.path.exists(self.clientPath):
1998 new_client_dir = True
1999 os.makedirs(self.clientPath)
2001 chdir(self.clientPath, is_client_path=True)
2003 print "Would synchronize p4 checkout in %s" % self.clientPath
2005 print "Synchronizing p4 checkout..."
2007 # old one was destroyed, and maybe nobody told p4
2008 p4_sync("...", "-f")
2015 commitish = self.master
2019 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2020 commits.append(line.strip())
2023 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2024 self.checkAuthorship = False
2026 self.checkAuthorship = True
2028 if self.preserveUser:
2029 self.checkValidP4Users(commits)
2032 # Build up a set of options to be passed to diff when
2033 # submitting each commit to p4.
2035 if self.detectRenames:
2036 # command-line -M arg
2037 self.diffOpts = "-M"
2039 # If not explicitly set check the config variable
2040 detectRenames = gitConfig("git-p4.detectRenames")
2042 if detectRenames.lower() == "false" or detectRenames == "":
2044 elif detectRenames.lower() == "true":
2045 self.diffOpts = "-M"
2047 self.diffOpts = "-M%s" % detectRenames
2049 # no command-line arg for -C or --find-copies-harder, just
2051 detectCopies = gitConfig("git-p4.detectCopies")
2052 if detectCopies.lower() == "false" or detectCopies == "":
2054 elif detectCopies.lower() == "true":
2055 self.diffOpts += " -C"
2057 self.diffOpts += " -C%s" % detectCopies
2059 if gitConfigBool("git-p4.detectCopiesHarder"):
2060 self.diffOpts += " --find-copies-harder"
2063 # Apply the commits, one at a time. On failure, ask if should
2064 # continue to try the rest of the patches, or quit.
2069 last = len(commits) - 1
2070 for i, commit in enumerate(commits):
2072 print " ", read_pipe(["git", "show", "-s",
2073 "--format=format:%h %s", commit])
2076 ok = self.applyCommit(commit)
2078 applied.append(commit)
2080 if self.prepare_p4_only and i < last:
2081 print "Processing only the first commit due to option" \
2082 " --prepare-p4-only"
2087 # prompt for what to do, or use the option/variable
2088 if self.conflict_behavior == "ask":
2089 print "What do you want to do?"
2090 response = raw_input("[s]kip this commit but apply"
2091 " the rest, or [q]uit? ")
2094 elif self.conflict_behavior == "skip":
2096 elif self.conflict_behavior == "quit":
2099 die("Unknown conflict_behavior '%s'" %
2100 self.conflict_behavior)
2102 if response[0] == "s":
2103 print "Skipping this commit, but applying the rest"
2105 if response[0] == "q":
2112 chdir(self.oldWorkingDirectory)
2113 shelved_applied = "shelved" if self.shelve else "applied"
2116 elif self.prepare_p4_only:
2118 elif len(commits) == len(applied):
2119 print ("All commits {0}!".format(shelved_applied))
2123 sync.branch = self.branch
2130 if len(applied) == 0:
2131 print ("No commits {0}.".format(shelved_applied))
2133 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2139 print star, read_pipe(["git", "show", "-s",
2140 "--format=format:%h %s", c])
2141 print "You will have to do 'git p4 sync' and rebase."
2143 if gitConfigBool("git-p4.exportLabels"):
2144 self.exportLabels = True
2146 if self.exportLabels:
2147 p4Labels = getP4Labels(self.depotPath)
2148 gitTags = getGitTags()
2150 missingGitTags = gitTags - p4Labels
2151 self.exportGitTags(missingGitTags)
2153 # exit with error unless everything applied perfectly
2154 if len(commits) != len(applied):
2160 """Represent a p4 view ("p4 help views"), and map files in a
2161 repo according to the view."""
2163 def __init__(self, client_name):
2165 self.client_prefix = "//%s/" % client_name
2166 # cache results of "p4 where" to lookup client file locations
2167 self.client_spec_path_cache = {}
2169 def append(self, view_line):
2170 """Parse a view line, splitting it into depot and client
2171 sides. Append to self.mappings, preserving order. This
2172 is only needed for tag creation."""
2174 # Split the view line into exactly two words. P4 enforces
2175 # structure on these lines that simplifies this quite a bit.
2177 # Either or both words may be double-quoted.
2178 # Single quotes do not matter.
2179 # Double-quote marks cannot occur inside the words.
2180 # A + or - prefix is also inside the quotes.
2181 # There are no quotes unless they contain a space.
2182 # The line is already white-space stripped.
2183 # The two words are separated by a single space.
2185 if view_line[0] == '"':
2186 # First word is double quoted. Find its end.
2187 close_quote_index = view_line.find('"', 1)
2188 if close_quote_index <= 0:
2189 die("No first-word closing quote found: %s" % view_line)
2190 depot_side = view_line[1:close_quote_index]
2191 # skip closing quote and space
2192 rhs_index = close_quote_index + 1 + 1
2194 space_index = view_line.find(" ")
2195 if space_index <= 0:
2196 die("No word-splitting space found: %s" % view_line)
2197 depot_side = view_line[0:space_index]
2198 rhs_index = space_index + 1
2200 # prefix + means overlay on previous mapping
2201 if depot_side.startswith("+"):
2202 depot_side = depot_side[1:]
2204 # prefix - means exclude this path, leave out of mappings
2206 if depot_side.startswith("-"):
2208 depot_side = depot_side[1:]
2211 self.mappings.append(depot_side)
2213 def convert_client_path(self, clientFile):
2214 # chop off //client/ part to make it relative
2215 if not clientFile.startswith(self.client_prefix):
2216 die("No prefix '%s' on clientFile '%s'" %
2217 (self.client_prefix, clientFile))
2218 return clientFile[len(self.client_prefix):]
2220 def update_client_spec_path_cache(self, files):
2221 """ Caching file paths by "p4 where" batch query """
2223 # List depot file paths exclude that already cached
2224 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2226 if len(fileArgs) == 0:
2227 return # All files in cache
2229 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2230 for res in where_result:
2231 if "code" in res and res["code"] == "error":
2232 # assume error is "... file(s) not in client view"
2234 if "clientFile" not in res:
2235 die("No clientFile in 'p4 where' output")
2237 # it will list all of them, but only one not unmap-ped
2239 if gitConfigBool("core.ignorecase"):
2240 res['depotFile'] = res['depotFile'].lower()
2241 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2243 # not found files or unmap files set to ""
2244 for depotFile in fileArgs:
2245 if gitConfigBool("core.ignorecase"):
2246 depotFile = depotFile.lower()
2247 if depotFile not in self.client_spec_path_cache:
2248 self.client_spec_path_cache[depotFile] = ""
2250 def map_in_client(self, depot_path):
2251 """Return the relative location in the client where this
2252 depot file should live. Returns "" if the file should
2253 not be mapped in the client."""
2255 if gitConfigBool("core.ignorecase"):
2256 depot_path = depot_path.lower()
2258 if depot_path in self.client_spec_path_cache:
2259 return self.client_spec_path_cache[depot_path]
2261 die( "Error: %s is not found in client spec path" % depot_path )
2264 class P4Sync(Command, P4UserMap):
2265 delete_actions = ( "delete", "move/delete", "purge" )
2268 Command.__init__(self)
2269 P4UserMap.__init__(self)
2271 optparse.make_option("--branch", dest="branch"),
2272 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2273 optparse.make_option("--changesfile", dest="changesFile"),
2274 optparse.make_option("--silent", dest="silent", action="store_true"),
2275 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2276 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2277 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2278 help="Import into refs/heads/ , not refs/remotes"),
2279 optparse.make_option("--max-changes", dest="maxChanges",
2280 help="Maximum number of changes to import"),
2281 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2282 help="Internal block size to use when iteratively calling p4 changes"),
2283 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2284 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2285 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2286 help="Only sync files that are included in the Perforce Client Spec"),
2287 optparse.make_option("-/", dest="cloneExclude",
2288 action="append", type="string",
2289 help="exclude depot path"),
2291 self.description = """Imports from Perforce into a git repository.\n
2293 //depot/my/project/ -- to import the current head
2294 //depot/my/project/@all -- to import everything
2295 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2297 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2299 self.usage += " //depot/path[@revRange]"
2301 self.createdBranches = set()
2302 self.committedChanges = set()
2304 self.detectBranches = False
2305 self.detectLabels = False
2306 self.importLabels = False
2307 self.changesFile = ""
2308 self.syncWithOrigin = True
2309 self.importIntoRemotes = True
2310 self.maxChanges = ""
2311 self.changes_block_size = None
2312 self.keepRepoPath = False
2313 self.depotPaths = None
2314 self.p4BranchesInGit = []
2315 self.cloneExclude = []
2316 self.useClientSpec = False
2317 self.useClientSpec_from_options = False
2318 self.clientSpecDirs = None
2319 self.tempBranches = []
2320 self.tempBranchLocation = "refs/git-p4-tmp"
2321 self.largeFileSystem = None
2323 if gitConfig('git-p4.largeFileSystem'):
2324 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2325 self.largeFileSystem = largeFileSystemConstructor(
2326 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2329 if gitConfig("git-p4.syncFromOrigin") == "false":
2330 self.syncWithOrigin = False
2332 # This is required for the "append" cloneExclude action
2333 def ensure_value(self, attr, value):
2334 if not hasattr(self, attr) or getattr(self, attr) is None:
2335 setattr(self, attr, value)
2336 return getattr(self, attr)
2338 # Force a checkpoint in fast-import and wait for it to finish
2339 def checkpoint(self):
2340 self.gitStream.write("checkpoint\n\n")
2341 self.gitStream.write("progress checkpoint\n\n")
2342 out = self.gitOutput.readline()
2344 print "checkpoint finished: " + out
2346 def extractFilesFromCommit(self, commit):
2347 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2348 for path in self.cloneExclude]
2351 while commit.has_key("depotFile%s" % fnum):
2352 path = commit["depotFile%s" % fnum]
2354 if [p for p in self.cloneExclude
2355 if p4PathStartsWith(path, p)]:
2358 found = [p for p in self.depotPaths
2359 if p4PathStartsWith(path, p)]
2366 file["rev"] = commit["rev%s" % fnum]
2367 file["action"] = commit["action%s" % fnum]
2368 file["type"] = commit["type%s" % fnum]
2373 def extractJobsFromCommit(self, commit):
2376 while commit.has_key("job%s" % jnum):
2377 job = commit["job%s" % jnum]
2382 def stripRepoPath(self, path, prefixes):
2383 """When streaming files, this is called to map a p4 depot path
2384 to where it should go in git. The prefixes are either
2385 self.depotPaths, or self.branchPrefixes in the case of
2386 branch detection."""
2388 if self.useClientSpec:
2389 # branch detection moves files up a level (the branch name)
2390 # from what client spec interpretation gives
2391 path = self.clientSpecDirs.map_in_client(path)
2392 if self.detectBranches:
2393 for b in self.knownBranches:
2394 if path.startswith(b + "/"):
2395 path = path[len(b)+1:]
2397 elif self.keepRepoPath:
2398 # Preserve everything in relative path name except leading
2399 # //depot/; just look at first prefix as they all should
2400 # be in the same depot.
2401 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2402 if p4PathStartsWith(path, depot):
2403 path = path[len(depot):]
2407 if p4PathStartsWith(path, p):
2408 path = path[len(p):]
2411 path = wildcard_decode(path)
2414 def splitFilesIntoBranches(self, commit):
2415 """Look at each depotFile in the commit to figure out to what
2416 branch it belongs."""
2418 if self.clientSpecDirs:
2419 files = self.extractFilesFromCommit(commit)
2420 self.clientSpecDirs.update_client_spec_path_cache(files)
2424 while commit.has_key("depotFile%s" % fnum):
2425 path = commit["depotFile%s" % fnum]
2426 found = [p for p in self.depotPaths
2427 if p4PathStartsWith(path, p)]
2434 file["rev"] = commit["rev%s" % fnum]
2435 file["action"] = commit["action%s" % fnum]
2436 file["type"] = commit["type%s" % fnum]
2439 # start with the full relative path where this file would
2441 if self.useClientSpec:
2442 relPath = self.clientSpecDirs.map_in_client(path)
2444 relPath = self.stripRepoPath(path, self.depotPaths)
2446 for branch in self.knownBranches.keys():
2447 # add a trailing slash so that a commit into qt/4.2foo
2448 # doesn't end up in qt/4.2, e.g.
2449 if relPath.startswith(branch + "/"):
2450 if branch not in branches:
2451 branches[branch] = []
2452 branches[branch].append(file)
2457 def writeToGitStream(self, gitMode, relPath, contents):
2458 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2459 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2461 self.gitStream.write(d)
2462 self.gitStream.write('\n')
2464 # output one file from the P4 stream
2465 # - helper for streamP4Files
2467 def streamOneP4File(self, file, contents):
2468 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2470 size = int(self.stream_file['fileSize'])
2471 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2474 (type_base, type_mods) = split_p4_type(file["type"])
2477 if "x" in type_mods:
2479 if type_base == "symlink":
2481 # p4 print on a symlink sometimes contains "target\n";
2482 # if it does, remove the newline
2483 data = ''.join(contents)
2485 # Some version of p4 allowed creating a symlink that pointed
2486 # to nothing. This causes p4 errors when checking out such
2487 # a change, and errors here too. Work around it by ignoring
2488 # the bad symlink; hopefully a future change fixes it.
2489 print "\nIgnoring empty symlink in %s" % file['depotFile']
2491 elif data[-1] == '\n':
2492 contents = [data[:-1]]
2496 if type_base == "utf16":
2497 # p4 delivers different text in the python output to -G
2498 # than it does when using "print -o", or normal p4 client
2499 # operations. utf16 is converted to ascii or utf8, perhaps.
2500 # But ascii text saved as -t utf16 is completely mangled.
2501 # Invoke print -o to get the real contents.
2503 # On windows, the newlines will always be mangled by print, so put
2504 # them back too. This is not needed to the cygwin windows version,
2505 # just the native "NT" type.
2508 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2509 except Exception as e:
2510 if 'Translation of file content failed' in str(e):
2511 type_base = 'binary'
2515 if p4_version_string().find('/NT') >= 0:
2516 text = text.replace('\r\n', '\n')
2519 if type_base == "apple":
2520 # Apple filetype files will be streamed as a concatenation of
2521 # its appledouble header and the contents. This is useless
2522 # on both macs and non-macs. If using "print -q -o xx", it
2523 # will create "xx" with the data, and "%xx" with the header.
2524 # This is also not very useful.
2526 # Ideally, someday, this script can learn how to generate
2527 # appledouble files directly and import those to git, but
2528 # non-mac machines can never find a use for apple filetype.
2529 print "\nIgnoring apple filetype file %s" % file['depotFile']
2532 # Note that we do not try to de-mangle keywords on utf16 files,
2533 # even though in theory somebody may want that.
2534 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2536 regexp = re.compile(pattern, re.VERBOSE)
2537 text = ''.join(contents)
2538 text = regexp.sub(r'$\1$', text)
2542 relPath.decode('ascii')
2545 if gitConfig('git-p4.pathEncoding'):
2546 encoding = gitConfig('git-p4.pathEncoding')
2547 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2549 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2551 if self.largeFileSystem:
2552 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2554 self.writeToGitStream(git_mode, relPath, contents)
2556 def streamOneP4Deletion(self, file):
2557 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2559 sys.stdout.write("delete %s\n" % relPath)
2561 self.gitStream.write("D %s\n" % relPath)
2563 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2564 self.largeFileSystem.removeLargeFile(relPath)
2566 # handle another chunk of streaming data
2567 def streamP4FilesCb(self, marshalled):
2569 # catch p4 errors and complain
2571 if "code" in marshalled:
2572 if marshalled["code"] == "error":
2573 if "data" in marshalled:
2574 err = marshalled["data"].rstrip()
2576 if not err and 'fileSize' in self.stream_file:
2577 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2578 if required_bytes > 0:
2579 err = 'Not enough space left on %s! Free at least %i MB.' % (
2580 os.getcwd(), required_bytes/1024/1024
2585 if self.stream_have_file_info:
2586 if "depotFile" in self.stream_file:
2587 f = self.stream_file["depotFile"]
2588 # force a failure in fast-import, else an empty
2589 # commit will be made
2590 self.gitStream.write("\n")
2591 self.gitStream.write("die-now\n")
2592 self.gitStream.close()
2593 # ignore errors, but make sure it exits first
2594 self.importProcess.wait()
2596 die("Error from p4 print for %s: %s" % (f, err))
2598 die("Error from p4 print: %s" % err)
2600 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2601 # start of a new file - output the old one first
2602 self.streamOneP4File(self.stream_file, self.stream_contents)
2603 self.stream_file = {}
2604 self.stream_contents = []
2605 self.stream_have_file_info = False
2607 # pick up the new file information... for the
2608 # 'data' field we need to append to our array
2609 for k in marshalled.keys():
2611 if 'streamContentSize' not in self.stream_file:
2612 self.stream_file['streamContentSize'] = 0
2613 self.stream_file['streamContentSize'] += len(marshalled['data'])
2614 self.stream_contents.append(marshalled['data'])
2616 self.stream_file[k] = marshalled[k]
2619 'streamContentSize' in self.stream_file and
2620 'fileSize' in self.stream_file and
2621 'depotFile' in self.stream_file):
2622 size = int(self.stream_file["fileSize"])
2624 progress = 100*self.stream_file['streamContentSize']/size
2625 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2628 self.stream_have_file_info = True
2630 # Stream directly from "p4 files" into "git fast-import"
2631 def streamP4Files(self, files):
2637 filesForCommit.append(f)
2638 if f['action'] in self.delete_actions:
2639 filesToDelete.append(f)
2641 filesToRead.append(f)
2644 for f in filesToDelete:
2645 self.streamOneP4Deletion(f)
2647 if len(filesToRead) > 0:
2648 self.stream_file = {}
2649 self.stream_contents = []
2650 self.stream_have_file_info = False
2652 # curry self argument
2653 def streamP4FilesCbSelf(entry):
2654 self.streamP4FilesCb(entry)
2656 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2658 p4CmdList(["-x", "-", "print"],
2660 cb=streamP4FilesCbSelf)
2663 if self.stream_file.has_key('depotFile'):
2664 self.streamOneP4File(self.stream_file, self.stream_contents)
2666 def make_email(self, userid):
2667 if userid in self.users:
2668 return self.users[userid]
2670 return "%s <a@b>" % userid
2672 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2673 """ Stream a p4 tag.
2674 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2678 print "writing tag %s for commit %s" % (labelName, commit)
2679 gitStream.write("tag %s\n" % labelName)
2680 gitStream.write("from %s\n" % commit)
2682 if labelDetails.has_key('Owner'):
2683 owner = labelDetails["Owner"]
2687 # Try to use the owner of the p4 label, or failing that,
2688 # the current p4 user id.
2690 email = self.make_email(owner)
2692 email = self.make_email(self.p4UserId())
2693 tagger = "%s %s %s" % (email, epoch, self.tz)
2695 gitStream.write("tagger %s\n" % tagger)
2697 print "labelDetails=",labelDetails
2698 if labelDetails.has_key('Description'):
2699 description = labelDetails['Description']
2701 description = 'Label from git p4'
2703 gitStream.write("data %d\n" % len(description))
2704 gitStream.write(description)
2705 gitStream.write("\n")
2707 def inClientSpec(self, path):
2708 if not self.clientSpecDirs:
2710 inClientSpec = self.clientSpecDirs.map_in_client(path)
2711 if not inClientSpec and self.verbose:
2712 print('Ignoring file outside of client spec: {0}'.format(path))
2715 def hasBranchPrefix(self, path):
2716 if not self.branchPrefixes:
2718 hasPrefix = [p for p in self.branchPrefixes
2719 if p4PathStartsWith(path, p)]
2720 if not hasPrefix and self.verbose:
2721 print('Ignoring file outside of prefix: {0}'.format(path))
2724 def commit(self, details, files, branch, parent = ""):
2725 epoch = details["time"]
2726 author = details["user"]
2727 jobs = self.extractJobsFromCommit(details)
2730 print('commit into {0}'.format(branch))
2732 if self.clientSpecDirs:
2733 self.clientSpecDirs.update_client_spec_path_cache(files)
2735 files = [f for f in files
2736 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2738 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2739 print('Ignoring revision {0} as it would produce an empty commit.'
2740 .format(details['change']))
2743 self.gitStream.write("commit %s\n" % branch)
2744 self.gitStream.write("mark :%s\n" % details["change"])
2745 self.committedChanges.add(int(details["change"]))
2747 if author not in self.users:
2748 self.getUserMapFromPerforceServer()
2749 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2751 self.gitStream.write("committer %s\n" % committer)
2753 self.gitStream.write("data <<EOT\n")
2754 self.gitStream.write(details["desc"])
2756 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2757 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2758 (','.join(self.branchPrefixes), details["change"]))
2759 if len(details['options']) > 0:
2760 self.gitStream.write(": options = %s" % details['options'])
2761 self.gitStream.write("]\nEOT\n\n")
2765 print "parent %s" % parent
2766 self.gitStream.write("from %s\n" % parent)
2768 self.streamP4Files(files)
2769 self.gitStream.write("\n")
2771 change = int(details["change"])
2773 if self.labels.has_key(change):
2774 label = self.labels[change]
2775 labelDetails = label[0]
2776 labelRevisions = label[1]
2778 print "Change %s is labelled %s" % (change, labelDetails)
2780 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2781 for p in self.branchPrefixes])
2783 if len(files) == len(labelRevisions):
2787 if info["action"] in self.delete_actions:
2789 cleanedFiles[info["depotFile"]] = info["rev"]
2791 if cleanedFiles == labelRevisions:
2792 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2796 print ("Tag %s does not match with change %s: files do not match."
2797 % (labelDetails["label"], change))
2801 print ("Tag %s does not match with change %s: file count is different."
2802 % (labelDetails["label"], change))
2804 # Build a dictionary of changelists and labels, for "detect-labels" option.
2805 def getLabels(self):
2808 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2809 if len(l) > 0 and not self.silent:
2810 print "Finding files belonging to labels in %s" % `self.depotPaths`
2813 label = output["label"]
2817 print "Querying files for label %s" % label
2818 for file in p4CmdList(["files"] +
2819 ["%s...@%s" % (p, label)
2820 for p in self.depotPaths]):
2821 revisions[file["depotFile"]] = file["rev"]
2822 change = int(file["change"])
2823 if change > newestChange:
2824 newestChange = change
2826 self.labels[newestChange] = [output, revisions]
2829 print "Label changes: %s" % self.labels.keys()
2831 # Import p4 labels as git tags. A direct mapping does not
2832 # exist, so assume that if all the files are at the same revision
2833 # then we can use that, or it's something more complicated we should
2835 def importP4Labels(self, stream, p4Labels):
2837 print "import p4 labels: " + ' '.join(p4Labels)
2839 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2840 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2841 if len(validLabelRegexp) == 0:
2842 validLabelRegexp = defaultLabelRegexp
2843 m = re.compile(validLabelRegexp)
2845 for name in p4Labels:
2848 if not m.match(name):
2850 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2853 if name in ignoredP4Labels:
2856 labelDetails = p4CmdList(['label', "-o", name])[0]
2858 # get the most recent changelist for each file in this label
2859 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2860 for p in self.depotPaths])
2862 if change.has_key('change'):
2863 # find the corresponding git commit; take the oldest commit
2864 changelist = int(change['change'])
2865 if changelist in self.committedChanges:
2866 gitCommit = ":%d" % changelist # use a fast-import mark
2869 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2870 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2871 if len(gitCommit) == 0:
2872 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2875 gitCommit = gitCommit.strip()
2878 # Convert from p4 time format
2880 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2882 print "Could not convert label time %s" % labelDetails['Update']
2885 when = int(time.mktime(tmwhen))
2886 self.streamTag(stream, name, labelDetails, gitCommit, when)
2888 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2891 print "Label %s has no changelists - possibly deleted?" % name
2894 # We can't import this label; don't try again as it will get very
2895 # expensive repeatedly fetching all the files for labels that will
2896 # never be imported. If the label is moved in the future, the
2897 # ignore will need to be removed manually.
2898 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2900 def guessProjectName(self):
2901 for p in self.depotPaths:
2904 p = p[p.strip().rfind("/") + 1:]
2905 if not p.endswith("/"):
2909 def getBranchMapping(self):
2910 lostAndFoundBranches = set()
2912 user = gitConfig("git-p4.branchUser")
2914 command = "branches -u %s" % user
2916 command = "branches"
2918 for info in p4CmdList(command):
2919 details = p4Cmd(["branch", "-o", info["branch"]])
2921 while details.has_key("View%s" % viewIdx):
2922 paths = details["View%s" % viewIdx].split(" ")
2923 viewIdx = viewIdx + 1
2924 # require standard //depot/foo/... //depot/bar/... mapping
2925 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2928 destination = paths[1]
2930 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2931 source = source[len(self.depotPaths[0]):-4]
2932 destination = destination[len(self.depotPaths[0]):-4]
2934 if destination in self.knownBranches:
2936 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2937 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2940 self.knownBranches[destination] = source
2942 lostAndFoundBranches.discard(destination)
2944 if source not in self.knownBranches:
2945 lostAndFoundBranches.add(source)
2947 # Perforce does not strictly require branches to be defined, so we also
2948 # check git config for a branch list.
2950 # Example of branch definition in git config file:
2952 # branchList=main:branchA
2953 # branchList=main:branchB
2954 # branchList=branchA:branchC
2955 configBranches = gitConfigList("git-p4.branchList")
2956 for branch in configBranches:
2958 (source, destination) = branch.split(":")
2959 self.knownBranches[destination] = source
2961 lostAndFoundBranches.discard(destination)
2963 if source not in self.knownBranches:
2964 lostAndFoundBranches.add(source)
2967 for branch in lostAndFoundBranches:
2968 self.knownBranches[branch] = branch
2970 def getBranchMappingFromGitBranches(self):
2971 branches = p4BranchesInGit(self.importIntoRemotes)
2972 for branch in branches.keys():
2973 if branch == "master":
2976 branch = branch[len(self.projectName):]
2977 self.knownBranches[branch] = branch
2979 def updateOptionDict(self, d):
2981 if self.keepRepoPath:
2982 option_keys['keepRepoPath'] = 1
2984 d["options"] = ' '.join(sorted(option_keys.keys()))
2986 def readOptions(self, d):
2987 self.keepRepoPath = (d.has_key('options')
2988 and ('keepRepoPath' in d['options']))
2990 def gitRefForBranch(self, branch):
2991 if branch == "main":
2992 return self.refPrefix + "master"
2994 if len(branch) <= 0:
2997 return self.refPrefix + self.projectName + branch
2999 def gitCommitByP4Change(self, ref, change):
3001 print "looking in ref " + ref + " for change %s using bisect..." % change
3004 latestCommit = parseRevision(ref)
3008 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3009 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3014 log = extractLogMessageFromGitCommit(next)
3015 settings = extractSettingsGitLog(log)
3016 currentChange = int(settings['change'])
3018 print "current change %s" % currentChange
3020 if currentChange == change:
3022 print "found %s" % next
3025 if currentChange < change:
3026 earliestCommit = "^%s" % next
3028 latestCommit = "%s" % next
3032 def importNewBranch(self, branch, maxChange):
3033 # make fast-import flush all changes to disk and update the refs using the checkpoint
3034 # command so that we can try to find the branch parent in the git history
3035 self.gitStream.write("checkpoint\n\n");
3036 self.gitStream.flush();
3037 branchPrefix = self.depotPaths[0] + branch + "/"
3038 range = "@1,%s" % maxChange
3039 #print "prefix" + branchPrefix
3040 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3041 if len(changes) <= 0:
3043 firstChange = changes[0]
3044 #print "first change in branch: %s" % firstChange
3045 sourceBranch = self.knownBranches[branch]
3046 sourceDepotPath = self.depotPaths[0] + sourceBranch
3047 sourceRef = self.gitRefForBranch(sourceBranch)
3048 #print "source " + sourceBranch
3050 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3051 #print "branch parent: %s" % branchParentChange
3052 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3053 if len(gitParent) > 0:
3054 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3055 #print "parent git commit: %s" % gitParent
3057 self.importChanges(changes)
3060 def searchParent(self, parent, branch, target):
3062 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3063 "--no-merges", parent]):
3065 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3068 print "Found parent of %s in commit %s" % (branch, blob)
3075 def importChanges(self, changes):
3077 for change in changes:
3078 description = p4_describe(change)
3079 self.updateOptionDict(description)
3082 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3087 if self.detectBranches:
3088 branches = self.splitFilesIntoBranches(description)
3089 for branch in branches.keys():
3091 branchPrefix = self.depotPaths[0] + branch + "/"
3092 self.branchPrefixes = [ branchPrefix ]
3096 filesForCommit = branches[branch]
3099 print "branch is %s" % branch
3101 self.updatedBranches.add(branch)
3103 if branch not in self.createdBranches:
3104 self.createdBranches.add(branch)
3105 parent = self.knownBranches[branch]
3106 if parent == branch:
3109 fullBranch = self.projectName + branch
3110 if fullBranch not in self.p4BranchesInGit:
3112 print("\n Importing new branch %s" % fullBranch);
3113 if self.importNewBranch(branch, change - 1):
3115 self.p4BranchesInGit.append(fullBranch)
3117 print("\n Resuming with change %s" % change);
3120 print "parent determined through known branches: %s" % parent
3122 branch = self.gitRefForBranch(branch)
3123 parent = self.gitRefForBranch(parent)
3126 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3128 if len(parent) == 0 and branch in self.initialParents:
3129 parent = self.initialParents[branch]
3130 del self.initialParents[branch]
3134 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3136 print "Creating temporary branch: " + tempBranch
3137 self.commit(description, filesForCommit, tempBranch)
3138 self.tempBranches.append(tempBranch)
3140 blob = self.searchParent(parent, branch, tempBranch)
3142 self.commit(description, filesForCommit, branch, blob)
3145 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3146 self.commit(description, filesForCommit, branch, parent)
3148 files = self.extractFilesFromCommit(description)
3149 self.commit(description, files, self.branch,
3151 # only needed once, to connect to the previous commit
3152 self.initialParent = ""
3154 print self.gitError.read()
3157 def importHeadRevision(self, revision):
3158 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3161 details["user"] = "git perforce import user"
3162 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3163 % (' '.join(self.depotPaths), revision))
3164 details["change"] = revision
3168 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3170 for info in p4CmdList(["files"] + fileArgs):
3172 if 'code' in info and info['code'] == 'error':
3173 sys.stderr.write("p4 returned an error: %s\n"
3175 if info['data'].find("must refer to client") >= 0:
3176 sys.stderr.write("This particular p4 error is misleading.\n")
3177 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3178 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3180 if 'p4ExitCode' in info:
3181 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3185 change = int(info["change"])
3186 if change > newestRevision:
3187 newestRevision = change
3189 if info["action"] in self.delete_actions:
3190 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3191 #fileCnt = fileCnt + 1
3194 for prop in ["depotFile", "rev", "action", "type" ]:
3195 details["%s%s" % (prop, fileCnt)] = info[prop]
3197 fileCnt = fileCnt + 1
3199 details["change"] = newestRevision
3201 # Use time from top-most change so that all git p4 clones of
3202 # the same p4 repo have the same commit SHA1s.
3203 res = p4_describe(newestRevision)
3204 details["time"] = res["time"]
3206 self.updateOptionDict(details)
3208 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3210 print "IO error with git fast-import. Is your git version recent enough?"
3211 print self.gitError.read()
3214 def run(self, args):
3215 self.depotPaths = []
3216 self.changeRange = ""
3217 self.previousDepotPaths = []
3218 self.hasOrigin = False
3220 # map from branch depot path to parent branch
3221 self.knownBranches = {}
3222 self.initialParents = {}
3224 if self.importIntoRemotes:
3225 self.refPrefix = "refs/remotes/p4/"
3227 self.refPrefix = "refs/heads/p4/"
3229 if self.syncWithOrigin:
3230 self.hasOrigin = originP4BranchesExist()
3233 print 'Syncing with origin first, using "git fetch origin"'
3234 system("git fetch origin")
3236 branch_arg_given = bool(self.branch)
3237 if len(self.branch) == 0:
3238 self.branch = self.refPrefix + "master"
3239 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3240 system("git update-ref %s refs/heads/p4" % self.branch)
3241 system("git branch -D p4")
3243 # accept either the command-line option, or the configuration variable
3244 if self.useClientSpec:
3245 # will use this after clone to set the variable
3246 self.useClientSpec_from_options = True
3248 if gitConfigBool("git-p4.useclientspec"):
3249 self.useClientSpec = True
3250 if self.useClientSpec:
3251 self.clientSpecDirs = getClientSpec()
3253 # TODO: should always look at previous commits,
3254 # merge with previous imports, if possible.
3257 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3259 # branches holds mapping from branch name to sha1
3260 branches = p4BranchesInGit(self.importIntoRemotes)
3262 # restrict to just this one, disabling detect-branches
3263 if branch_arg_given:
3264 short = self.branch.split("/")[-1]
3265 if short in branches:
3266 self.p4BranchesInGit = [ short ]
3268 self.p4BranchesInGit = branches.keys()
3270 if len(self.p4BranchesInGit) > 1:
3272 print "Importing from/into multiple branches"
3273 self.detectBranches = True
3274 for branch in branches.keys():
3275 self.initialParents[self.refPrefix + branch] = \
3279 print "branches: %s" % self.p4BranchesInGit
3282 for branch in self.p4BranchesInGit:
3283 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3285 settings = extractSettingsGitLog(logMsg)
3287 self.readOptions(settings)
3288 if (settings.has_key('depot-paths')
3289 and settings.has_key ('change')):
3290 change = int(settings['change']) + 1
3291 p4Change = max(p4Change, change)
3293 depotPaths = sorted(settings['depot-paths'])
3294 if self.previousDepotPaths == []:
3295 self.previousDepotPaths = depotPaths
3298 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3299 prev_list = prev.split("/")
3300 cur_list = cur.split("/")
3301 for i in range(0, min(len(cur_list), len(prev_list))):
3302 if cur_list[i] <> prev_list[i]:
3306 paths.append ("/".join(cur_list[:i + 1]))
3308 self.previousDepotPaths = paths
3311 self.depotPaths = sorted(self.previousDepotPaths)
3312 self.changeRange = "@%s,#head" % p4Change
3313 if not self.silent and not self.detectBranches:
3314 print "Performing incremental import into %s git branch" % self.branch
3316 # accept multiple ref name abbreviations:
3317 # refs/foo/bar/branch -> use it exactly
3318 # p4/branch -> prepend refs/remotes/ or refs/heads/
3319 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3320 if not self.branch.startswith("refs/"):
3321 if self.importIntoRemotes:
3322 prepend = "refs/remotes/"
3324 prepend = "refs/heads/"
3325 if not self.branch.startswith("p4/"):
3327 self.branch = prepend + self.branch
3329 if len(args) == 0 and self.depotPaths:
3331 print "Depot paths: %s" % ' '.join(self.depotPaths)
3333 if self.depotPaths and self.depotPaths != args:
3334 print ("previous import used depot path %s and now %s was specified. "
3335 "This doesn't work!" % (' '.join (self.depotPaths),
3339 self.depotPaths = sorted(args)
3344 # Make sure no revision specifiers are used when --changesfile
3346 bad_changesfile = False
3347 if len(self.changesFile) > 0:
3348 for p in self.depotPaths:
3349 if p.find("@") >= 0 or p.find("#") >= 0:
3350 bad_changesfile = True
3353 die("Option --changesfile is incompatible with revision specifiers")
3356 for p in self.depotPaths:
3357 if p.find("@") != -1:
3358 atIdx = p.index("@")
3359 self.changeRange = p[atIdx:]
3360 if self.changeRange == "@all":
3361 self.changeRange = ""
3362 elif ',' not in self.changeRange:
3363 revision = self.changeRange
3364 self.changeRange = ""
3366 elif p.find("#") != -1:
3367 hashIdx = p.index("#")
3368 revision = p[hashIdx:]
3370 elif self.previousDepotPaths == []:
3371 # pay attention to changesfile, if given, else import
3372 # the entire p4 tree at the head revision
3373 if len(self.changesFile) == 0:
3376 p = re.sub ("\.\.\.$", "", p)
3377 if not p.endswith("/"):
3382 self.depotPaths = newPaths
3384 # --detect-branches may change this for each branch
3385 self.branchPrefixes = self.depotPaths
3387 self.loadUserMapFromCache()
3389 if self.detectLabels:
3392 if self.detectBranches:
3393 ## FIXME - what's a P4 projectName ?
3394 self.projectName = self.guessProjectName()
3397 self.getBranchMappingFromGitBranches()
3399 self.getBranchMapping()
3401 print "p4-git branches: %s" % self.p4BranchesInGit
3402 print "initial parents: %s" % self.initialParents
3403 for b in self.p4BranchesInGit:
3407 b = b[len(self.projectName):]
3408 self.createdBranches.add(b)
3410 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3412 self.importProcess = subprocess.Popen(["git", "fast-import"],
3413 stdin=subprocess.PIPE,
3414 stdout=subprocess.PIPE,
3415 stderr=subprocess.PIPE);
3416 self.gitOutput = self.importProcess.stdout
3417 self.gitStream = self.importProcess.stdin
3418 self.gitError = self.importProcess.stderr
3421 self.importHeadRevision(revision)
3425 if len(self.changesFile) > 0:
3426 output = open(self.changesFile).readlines()
3429 changeSet.add(int(line))
3431 for change in changeSet:
3432 changes.append(change)
3436 # catch "git p4 sync" with no new branches, in a repo that
3437 # does not have any existing p4 branches
3439 if not self.p4BranchesInGit:
3440 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3442 # The default branch is master, unless --branch is used to
3443 # specify something else. Make sure it exists, or complain
3444 # nicely about how to use --branch.
3445 if not self.detectBranches:
3446 if not branch_exists(self.branch):
3447 if branch_arg_given:
3448 die("Error: branch %s does not exist." % self.branch)
3450 die("Error: no branch %s; perhaps specify one with --branch." %
3454 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3456 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3458 if len(self.maxChanges) > 0:
3459 changes = changes[:min(int(self.maxChanges), len(changes))]
3461 if len(changes) == 0:
3463 print "No changes to import!"
3465 if not self.silent and not self.detectBranches:
3466 print "Import destination: %s" % self.branch
3468 self.updatedBranches = set()
3470 if not self.detectBranches:
3472 # start a new branch
3473 self.initialParent = ""
3475 # build on a previous revision
3476 self.initialParent = parseRevision(self.branch)
3478 self.importChanges(changes)
3482 if len(self.updatedBranches) > 0:
3483 sys.stdout.write("Updated branches: ")
3484 for b in self.updatedBranches:
3485 sys.stdout.write("%s " % b)
3486 sys.stdout.write("\n")
3488 if gitConfigBool("git-p4.importLabels"):
3489 self.importLabels = True
3491 if self.importLabels:
3492 p4Labels = getP4Labels(self.depotPaths)
3493 gitTags = getGitTags()
3495 missingP4Labels = p4Labels - gitTags
3496 self.importP4Labels(self.gitStream, missingP4Labels)
3498 self.gitStream.close()
3499 if self.importProcess.wait() != 0:
3500 die("fast-import failed: %s" % self.gitError.read())
3501 self.gitOutput.close()
3502 self.gitError.close()
3504 # Cleanup temporary branches created during import
3505 if self.tempBranches != []:
3506 for branch in self.tempBranches:
3507 read_pipe("git update-ref -d %s" % branch)
3508 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3510 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3511 # a convenient shortcut refname "p4".
3512 if self.importIntoRemotes:
3513 head_ref = self.refPrefix + "HEAD"
3514 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3515 system(["git", "symbolic-ref", head_ref, self.branch])
3519 class P4Rebase(Command):
3521 Command.__init__(self)
3523 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3525 self.importLabels = False
3526 self.description = ("Fetches the latest revision from perforce and "
3527 + "rebases the current work (branch) against it")
3529 def run(self, args):
3531 sync.importLabels = self.importLabels
3534 return self.rebase()
3537 if os.system("git update-index --refresh") != 0:
3538 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3539 if len(read_pipe("git diff-index HEAD --")) > 0:
3540 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3542 [upstream, settings] = findUpstreamBranchPoint()
3543 if len(upstream) == 0:
3544 die("Cannot find upstream branchpoint for rebase")
3546 # the branchpoint may be p4/foo~3, so strip off the parent
3547 upstream = re.sub("~[0-9]+$", "", upstream)
3549 print "Rebasing the current branch onto %s" % upstream
3550 oldHead = read_pipe("git rev-parse HEAD").strip()
3551 system("git rebase %s" % upstream)
3552 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3555 class P4Clone(P4Sync):
3557 P4Sync.__init__(self)
3558 self.description = "Creates a new git repository and imports from Perforce into it"
3559 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3561 optparse.make_option("--destination", dest="cloneDestination",
3562 action='store', default=None,
3563 help="where to leave result of the clone"),
3564 optparse.make_option("--bare", dest="cloneBare",
3565 action="store_true", default=False),
3567 self.cloneDestination = None
3568 self.needsGit = False
3569 self.cloneBare = False
3571 def defaultDestination(self, args):
3572 ## TODO: use common prefix of args?
3574 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3575 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3576 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3577 depotDir = re.sub(r"/$", "", depotDir)
3578 return os.path.split(depotDir)[1]
3580 def run(self, args):
3584 if self.keepRepoPath and not self.cloneDestination:
3585 sys.stderr.write("Must specify destination for --keep-path\n")
3590 if not self.cloneDestination and len(depotPaths) > 1:
3591 self.cloneDestination = depotPaths[-1]
3592 depotPaths = depotPaths[:-1]
3594 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3595 for p in depotPaths:
3596 if not p.startswith("//"):
3597 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3600 if not self.cloneDestination:
3601 self.cloneDestination = self.defaultDestination(args)
3603 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3605 if not os.path.exists(self.cloneDestination):
3606 os.makedirs(self.cloneDestination)
3607 chdir(self.cloneDestination)
3609 init_cmd = [ "git", "init" ]
3611 init_cmd.append("--bare")
3612 retcode = subprocess.call(init_cmd)
3614 raise CalledProcessError(retcode, init_cmd)
3616 if not P4Sync.run(self, depotPaths):
3619 # create a master branch and check out a work tree
3620 if gitBranchExists(self.branch):
3621 system([ "git", "branch", "master", self.branch ])
3622 if not self.cloneBare:
3623 system([ "git", "checkout", "-f" ])
3625 print 'Not checking out any branch, use ' \
3626 '"git checkout -q -b master <branch>"'
3628 # auto-set this variable if invoked with --use-client-spec
3629 if self.useClientSpec_from_options:
3630 system("git config --bool git-p4.useclientspec true")
3634 class P4Branches(Command):
3636 Command.__init__(self)
3638 self.description = ("Shows the git branches that hold imports and their "
3639 + "corresponding perforce depot paths")
3640 self.verbose = False
3642 def run(self, args):
3643 if originP4BranchesExist():
3644 createOrUpdateBranchesFromOrigin()
3646 cmdline = "git rev-parse --symbolic "
3647 cmdline += " --remotes"
3649 for line in read_pipe_lines(cmdline):
3652 if not line.startswith('p4/') or line == "p4/HEAD":
3656 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3657 settings = extractSettingsGitLog(log)
3659 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3662 class HelpFormatter(optparse.IndentedHelpFormatter):
3664 optparse.IndentedHelpFormatter.__init__(self)
3666 def format_description(self, description):
3668 return description + "\n"
3672 def printUsage(commands):
3673 print "usage: %s <command> [options]" % sys.argv[0]
3675 print "valid commands: %s" % ", ".join(commands)
3677 print "Try %s <command> --help for command specific help." % sys.argv[0]
3682 "submit" : P4Submit,
3683 "commit" : P4Submit,
3685 "rebase" : P4Rebase,
3687 "rollback" : P4RollBack,
3688 "branches" : P4Branches
3693 if len(sys.argv[1:]) == 0:
3694 printUsage(commands.keys())
3697 cmdName = sys.argv[1]
3699 klass = commands[cmdName]
3702 print "unknown command %s" % cmdName
3704 printUsage(commands.keys())
3707 options = cmd.options
3708 cmd.gitdir = os.environ.get("GIT_DIR", None)
3712 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3714 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3716 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3718 description = cmd.description,
3719 formatter = HelpFormatter())
3721 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3723 verbose = cmd.verbose
3725 if cmd.gitdir == None:
3726 cmd.gitdir = os.path.abspath(".git")
3727 if not isValidGitDir(cmd.gitdir):
3728 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3729 if os.path.exists(cmd.gitdir):
3730 cdup = read_pipe("git rev-parse --show-cdup").strip()
3734 if not isValidGitDir(cmd.gitdir):
3735 if isValidGitDir(cmd.gitdir + "/.git"):
3736 cmd.gitdir += "/.git"
3738 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3740 os.environ["GIT_DIR"] = cmd.gitdir
3742 if not cmd.run(args):
3747 if __name__ == '__main__':