3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self, returncode, cmd):
39 self.returncode = returncode
42 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
52 def p4_build_cmd(cmd):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user = gitConfig("git-p4.user")
63 real_cmd += ["-u",user]
65 password = gitConfig("git-p4.password")
67 real_cmd += ["-P", password]
69 port = gitConfig("git-p4.port")
71 real_cmd += ["-p", port]
73 host = gitConfig("git-p4.host")
75 real_cmd += ["-H", host]
77 client = gitConfig("git-p4.client")
79 real_cmd += ["-c", client]
81 retries = gitConfigInt("git-p4.retries")
83 # Perform 3 retries by default
85 real_cmd += ["-r", str(retries)]
87 if isinstance(cmd,basestring):
88 real_cmd = ' '.join(real_cmd) + ' ' + cmd
94 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
95 This won't automatically add ".git" to a directory.
97 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
98 if not d or len(d) == 0:
103 def chdir(path, is_client_path=False):
104 """Do chdir to the given path, and set the PWD environment
105 variable for use by P4. It does not look at getcwd() output.
106 Since we're not using the shell, it is necessary to set the
107 PWD environment variable explicitly.
109 Normally, expand the path to force it to be absolute. This
110 addresses the use of relative path names inside P4 settings,
111 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
112 as given; it looks for .p4config using PWD.
114 If is_client_path, the path was handed to us directly by p4,
115 and may be a symbolic link. Do not call os.getcwd() in this
116 case, because it will cause p4 to think that PWD is not inside
121 if not is_client_path:
123 os.environ['PWD'] = path
126 """Return free space in bytes on the disk of the given dirname."""
127 if platform.system() == 'Windows':
128 free_bytes = ctypes.c_ulonglong(0)
129 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
130 return free_bytes.value
132 st = os.statvfs(os.getcwd())
133 return st.f_bavail * st.f_frsize
139 sys.stderr.write(msg + "\n")
142 def write_pipe(c, stdin):
144 sys.stderr.write('Writing pipe: %s\n' % str(c))
146 expand = isinstance(c,basestring)
147 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
149 val = pipe.write(stdin)
152 die('Command failed: %s' % str(c))
156 def p4_write_pipe(c, stdin):
157 real_cmd = p4_build_cmd(c)
158 return write_pipe(real_cmd, stdin)
160 def read_pipe(c, ignore_error=False):
162 sys.stderr.write('Reading pipe: %s\n' % str(c))
164 expand = isinstance(c,basestring)
165 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
166 (out, err) = p.communicate()
167 if p.returncode != 0 and not ignore_error:
168 die('Command failed: %s\nError: %s' % (str(c), err))
171 def p4_read_pipe(c, ignore_error=False):
172 real_cmd = p4_build_cmd(c)
173 return read_pipe(real_cmd, ignore_error)
175 def read_pipe_lines(c):
177 sys.stderr.write('Reading pipe: %s\n' % str(c))
179 expand = isinstance(c, basestring)
180 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
182 val = pipe.readlines()
183 if pipe.close() or p.wait():
184 die('Command failed: %s' % str(c))
188 def p4_read_pipe_lines(c):
189 """Specifically invoke p4 on the command supplied. """
190 real_cmd = p4_build_cmd(c)
191 return read_pipe_lines(real_cmd)
193 def p4_has_command(cmd):
194 """Ask p4 for help on this command. If it returns an error, the
195 command does not exist in this version of p4."""
196 real_cmd = p4_build_cmd(["help", cmd])
197 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
198 stderr=subprocess.PIPE)
200 return p.returncode == 0
202 def p4_has_move_command():
203 """See if the move command exists, that it supports -k, and that
204 it has not been administratively disabled. The arguments
205 must be correct, but the filenames do not have to exist. Use
206 ones with wildcards so even if they exist, it will fail."""
208 if not p4_has_command("move"):
210 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
211 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
212 (out, err) = p.communicate()
213 # return code will be 1 in either case
214 if err.find("Invalid option") >= 0:
216 if err.find("disabled") >= 0:
218 # assume it failed because @... was invalid changelist
221 def system(cmd, ignore_error=False):
222 expand = isinstance(cmd,basestring)
224 sys.stderr.write("executing %s\n" % str(cmd))
225 retcode = subprocess.call(cmd, shell=expand)
226 if retcode and not ignore_error:
227 raise CalledProcessError(retcode, cmd)
232 """Specifically invoke p4 as the system command. """
233 real_cmd = p4_build_cmd(cmd)
234 expand = isinstance(real_cmd, basestring)
235 retcode = subprocess.call(real_cmd, shell=expand)
237 raise CalledProcessError(retcode, real_cmd)
239 _p4_version_string = None
240 def p4_version_string():
241 """Read the version string, showing just the last line, which
242 hopefully is the interesting version bit.
245 Perforce - The Fast Software Configuration Management System.
246 Copyright 1995-2011 Perforce Software. All rights reserved.
247 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
249 global _p4_version_string
250 if not _p4_version_string:
251 a = p4_read_pipe_lines(["-V"])
252 _p4_version_string = a[-1].rstrip()
253 return _p4_version_string
255 def p4_integrate(src, dest):
256 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
258 def p4_sync(f, *options):
259 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
262 # forcibly add file names with wildcards
263 if wildcard_present(f):
264 p4_system(["add", "-f", f])
266 p4_system(["add", f])
269 p4_system(["delete", wildcard_encode(f)])
271 def p4_edit(f, *options):
272 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
275 p4_system(["revert", wildcard_encode(f)])
277 def p4_reopen(type, f):
278 p4_system(["reopen", "-t", type, wildcard_encode(f)])
280 def p4_reopen_in_change(changelist, files):
281 cmd = ["reopen", "-c", str(changelist)] + files
284 def p4_move(src, dest):
285 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
287 def p4_last_change():
288 results = p4CmdList(["changes", "-m", "1"])
289 return int(results[0]['change'])
291 def p4_describe(change):
292 """Make sure it returns a valid result by checking for
293 the presence of field "time". Return a dict of the
296 ds = p4CmdList(["describe", "-s", str(change)])
298 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
302 if "p4ExitCode" in d:
303 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
306 if d["code"] == "error":
307 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
310 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
315 # Canonicalize the p4 type and return a tuple of the
316 # base type, plus any modifiers. See "p4 help filetypes"
317 # for a list and explanation.
319 def split_p4_type(p4type):
321 p4_filetypes_historical = {
322 "ctempobj": "binary+Sw",
328 "tempobj": "binary+FSw",
329 "ubinary": "binary+F",
330 "uresource": "resource+F",
331 "uxbinary": "binary+Fx",
332 "xbinary": "binary+x",
334 "xtempobj": "binary+Swx",
336 "xunicode": "unicode+x",
339 if p4type in p4_filetypes_historical:
340 p4type = p4_filetypes_historical[p4type]
342 s = p4type.split("+")
350 # return the raw p4 type of a file (text, text+ko, etc)
353 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
354 return results[0]['headType']
357 # Given a type base and modifier, return a regexp matching
358 # the keywords that can be expanded in the file
360 def p4_keywords_regexp_for_type(base, type_mods):
361 if base in ("text", "unicode", "binary"):
363 if "ko" in type_mods:
365 elif "k" in type_mods:
366 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
370 \$ # Starts with a dollar, followed by...
371 (%s) # one of the keywords, followed by...
372 (:[^$\n]+)? # possibly an old expansion, followed by...
380 # Given a file, return a regexp matching the possible
381 # RCS keywords that will be expanded, or None for files
382 # with kw expansion turned off.
384 def p4_keywords_regexp_for_file(file):
385 if not os.path.exists(file):
388 (type_base, type_mods) = split_p4_type(p4_type(file))
389 return p4_keywords_regexp_for_type(type_base, type_mods)
391 def setP4ExecBit(file, mode):
392 # Reopens an already open file and changes the execute bit to match
393 # the execute bit setting in the passed in mode.
397 if not isModeExec(mode):
398 p4Type = getP4OpenedType(file)
399 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
400 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
401 if p4Type[-1] == "+":
402 p4Type = p4Type[0:-1]
404 p4_reopen(p4Type, file)
406 def getP4OpenedType(file):
407 # Returns the perforce file type for the given file.
409 result = p4_read_pipe(["opened", wildcard_encode(file)])
410 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
412 return match.group(1)
414 die("Could not determine file type for %s (result: '%s')" % (file, result))
416 # Return the set of all p4 labels
417 def getP4Labels(depotPaths):
419 if isinstance(depotPaths,basestring):
420 depotPaths = [depotPaths]
422 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
428 # Return the set of all git tags
431 for line in read_pipe_lines(["git", "tag"]):
436 def diffTreePattern():
437 # This is a simple generator for the diff tree regex pattern. This could be
438 # a class variable if this and parseDiffTreeEntry were a part of a class.
439 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
443 def parseDiffTreeEntry(entry):
444 """Parses a single diff tree entry into its component elements.
446 See git-diff-tree(1) manpage for details about the format of the diff
447 output. This method returns a dictionary with the following elements:
449 src_mode - The mode of the source file
450 dst_mode - The mode of the destination file
451 src_sha1 - The sha1 for the source file
452 dst_sha1 - The sha1 fr the destination file
453 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
454 status_score - The score for the status (applicable for 'C' and 'R'
455 statuses). This is None if there is no score.
456 src - The path for the source file.
457 dst - The path for the destination file. This is only present for
458 copy or renames. If it is not present, this is None.
460 If the pattern is not matched, None is returned."""
462 match = diffTreePattern().next().match(entry)
465 'src_mode': match.group(1),
466 'dst_mode': match.group(2),
467 'src_sha1': match.group(3),
468 'dst_sha1': match.group(4),
469 'status': match.group(5),
470 'status_score': match.group(6),
471 'src': match.group(7),
472 'dst': match.group(10)
476 def isModeExec(mode):
477 # Returns True if the given git mode represents an executable file,
479 return mode[-3:] == "755"
481 def isModeExecChanged(src_mode, dst_mode):
482 return isModeExec(src_mode) != isModeExec(dst_mode)
484 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
486 if isinstance(cmd,basestring):
493 cmd = p4_build_cmd(cmd)
495 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
497 # Use a temporary file to avoid deadlocks without
498 # subprocess.communicate(), which would put another copy
499 # of stdout into memory.
501 if stdin is not None:
502 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
503 if isinstance(stdin,basestring):
504 stdin_file.write(stdin)
507 stdin_file.write(i + '\n')
511 p4 = subprocess.Popen(cmd,
514 stdout=subprocess.PIPE)
519 entry = marshal.load(p4.stdout)
529 entry["p4ExitCode"] = exitCode
535 list = p4CmdList(cmd)
541 def p4Where(depotPath):
542 if not depotPath.endswith("/"):
544 depotPathLong = depotPath + "..."
545 outputList = p4CmdList(["where", depotPathLong])
547 for entry in outputList:
548 if "depotFile" in entry:
549 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
550 # The base path always ends with "/...".
551 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
554 elif "data" in entry:
555 data = entry.get("data")
556 space = data.find(" ")
557 if data[:space] == depotPath:
562 if output["code"] == "error":
566 clientPath = output.get("path")
567 elif "data" in output:
568 data = output.get("data")
569 lastSpace = data.rfind(" ")
570 clientPath = data[lastSpace + 1:]
572 if clientPath.endswith("..."):
573 clientPath = clientPath[:-3]
576 def currentGitBranch():
577 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
582 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
584 def isValidGitDir(path):
585 return git_dir(path) != None
587 def parseRevision(ref):
588 return read_pipe("git rev-parse %s" % ref).strip()
590 def branchExists(ref):
591 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
595 def extractLogMessageFromGitCommit(commit):
598 ## fixme: title is first line of commit, not 1st paragraph.
600 for log in read_pipe_lines("git cat-file commit %s" % commit):
609 def extractSettingsGitLog(log):
611 for line in log.split("\n"):
613 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
617 assignments = m.group(1).split (':')
618 for a in assignments:
620 key = vals[0].strip()
621 val = ('='.join (vals[1:])).strip()
622 if val.endswith ('\"') and val.startswith('"'):
627 paths = values.get("depot-paths")
629 paths = values.get("depot-path")
631 values['depot-paths'] = paths.split(',')
634 def gitBranchExists(branch):
635 proc = subprocess.Popen(["git", "rev-parse", branch],
636 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
637 return proc.wait() == 0;
641 def gitConfig(key, typeSpecifier=None):
642 if not _gitConfig.has_key(key):
643 cmd = [ "git", "config" ]
645 cmd += [ typeSpecifier ]
647 s = read_pipe(cmd, ignore_error=True)
648 _gitConfig[key] = s.strip()
649 return _gitConfig[key]
651 def gitConfigBool(key):
652 """Return a bool, using git config --bool. It is True only if the
653 variable is set to true, and False if set to false or not present
656 if not _gitConfig.has_key(key):
657 _gitConfig[key] = gitConfig(key, '--bool') == "true"
658 return _gitConfig[key]
660 def gitConfigInt(key):
661 if not _gitConfig.has_key(key):
662 cmd = [ "git", "config", "--int", key ]
663 s = read_pipe(cmd, ignore_error=True)
666 _gitConfig[key] = int(gitConfig(key, '--int'))
668 _gitConfig[key] = None
669 return _gitConfig[key]
671 def gitConfigList(key):
672 if not _gitConfig.has_key(key):
673 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
674 _gitConfig[key] = s.strip().split(os.linesep)
675 if _gitConfig[key] == ['']:
677 return _gitConfig[key]
679 def p4BranchesInGit(branchesAreInRemotes=True):
680 """Find all the branches whose names start with "p4/", looking
681 in remotes or heads as specified by the argument. Return
682 a dictionary of { branch: revision } for each one found.
683 The branch names are the short names, without any
688 cmdline = "git rev-parse --symbolic "
689 if branchesAreInRemotes:
690 cmdline += "--remotes"
692 cmdline += "--branches"
694 for line in read_pipe_lines(cmdline):
698 if not line.startswith('p4/'):
700 # special symbolic ref to p4/master
701 if line == "p4/HEAD":
704 # strip off p4/ prefix
705 branch = line[len("p4/"):]
707 branches[branch] = parseRevision(line)
711 def branch_exists(branch):
712 """Make sure that the given ref name really exists."""
714 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
715 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
716 out, _ = p.communicate()
719 # expect exactly one line of output: the branch name
720 return out.rstrip() == branch
722 def findUpstreamBranchPoint(head = "HEAD"):
723 branches = p4BranchesInGit()
724 # map from depot-path to branch name
725 branchByDepotPath = {}
726 for branch in branches.keys():
727 tip = branches[branch]
728 log = extractLogMessageFromGitCommit(tip)
729 settings = extractSettingsGitLog(log)
730 if settings.has_key("depot-paths"):
731 paths = ",".join(settings["depot-paths"])
732 branchByDepotPath[paths] = "remotes/p4/" + branch
736 while parent < 65535:
737 commit = head + "~%s" % parent
738 log = extractLogMessageFromGitCommit(commit)
739 settings = extractSettingsGitLog(log)
740 if settings.has_key("depot-paths"):
741 paths = ",".join(settings["depot-paths"])
742 if branchByDepotPath.has_key(paths):
743 return [branchByDepotPath[paths], settings]
747 return ["", settings]
749 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
751 print ("Creating/updating branch(es) in %s based on origin branch(es)"
754 originPrefix = "origin/p4/"
756 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
758 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
761 headName = line[len(originPrefix):]
762 remoteHead = localRefPrefix + headName
765 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
766 if (not original.has_key('depot-paths')
767 or not original.has_key('change')):
771 if not gitBranchExists(remoteHead):
773 print "creating %s" % remoteHead
776 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
777 if settings.has_key('change') > 0:
778 if settings['depot-paths'] == original['depot-paths']:
779 originP4Change = int(original['change'])
780 p4Change = int(settings['change'])
781 if originP4Change > p4Change:
782 print ("%s (%s) is newer than %s (%s). "
783 "Updating p4 branch from origin."
784 % (originHead, originP4Change,
785 remoteHead, p4Change))
788 print ("Ignoring: %s was imported from %s while "
789 "%s was imported from %s"
790 % (originHead, ','.join(original['depot-paths']),
791 remoteHead, ','.join(settings['depot-paths'])))
794 system("git update-ref %s %s" % (remoteHead, originHead))
796 def originP4BranchesExist():
797 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
800 def p4ParseNumericChangeRange(parts):
801 changeStart = int(parts[0][1:])
802 if parts[1] == '#head':
803 changeEnd = p4_last_change()
805 changeEnd = int(parts[1])
807 return (changeStart, changeEnd)
809 def chooseBlockSize(blockSize):
813 return defaultBlockSize
815 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
818 # Parse the change range into start and end. Try to find integer
819 # revision ranges as these can be broken up into blocks to avoid
820 # hitting server-side limits (maxrows, maxscanresults). But if
821 # that doesn't work, fall back to using the raw revision specifier
822 # strings, without using block mode.
824 if changeRange is None or changeRange == '':
826 changeEnd = p4_last_change()
827 block_size = chooseBlockSize(requestedBlockSize)
829 parts = changeRange.split(',')
830 assert len(parts) == 2
832 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
833 block_size = chooseBlockSize(requestedBlockSize)
835 changeStart = parts[0][1:]
837 if requestedBlockSize:
838 die("cannot use --changes-block-size with non-numeric revisions")
843 # Retrieve changes a block at a time, to prevent running
844 # into a MaxResults/MaxScanRows error from the server.
850 end = min(changeEnd, changeStart + block_size)
851 revisionRange = "%d,%d" % (changeStart, end)
853 revisionRange = "%s,%s" % (changeStart, changeEnd)
856 cmd += ["%s...@%s" % (p, revisionRange)]
858 # Insert changes in chronological order
859 for line in reversed(p4_read_pipe_lines(cmd)):
860 changes.append(int(line.split(" ")[1]))
868 changeStart = end + 1
870 changes = sorted(changes)
873 def p4PathStartsWith(path, prefix):
874 # This method tries to remedy a potential mixed-case issue:
876 # If UserA adds //depot/DirA/file1
877 # and UserB adds //depot/dira/file2
879 # we may or may not have a problem. If you have core.ignorecase=true,
880 # we treat DirA and dira as the same directory
881 if gitConfigBool("core.ignorecase"):
882 return path.lower().startswith(prefix.lower())
883 return path.startswith(prefix)
886 """Look at the p4 client spec, create a View() object that contains
887 all the mappings, and return it."""
889 specList = p4CmdList("client -o")
890 if len(specList) != 1:
891 die('Output from "client -o" is %d lines, expecting 1' %
894 # dictionary of all client parameters
898 client_name = entry["Client"]
900 # just the keys that start with "View"
901 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
904 view = View(client_name)
906 # append the lines, in order, to the view
907 for view_num in range(len(view_keys)):
908 k = "View%d" % view_num
909 if k not in view_keys:
910 die("Expected view key %s missing" % k)
911 view.append(entry[k])
916 """Grab the client directory."""
918 output = p4CmdList("client -o")
920 die('Output from "client -o" is %d lines, expecting 1' % len(output))
923 if "Root" not in entry:
924 die('Client has no "Root"')
929 # P4 wildcards are not allowed in filenames. P4 complains
930 # if you simply add them, but you can force it with "-f", in
931 # which case it translates them into %xx encoding internally.
933 def wildcard_decode(path):
934 # Search for and fix just these four characters. Do % last so
935 # that fixing it does not inadvertently create new %-escapes.
936 # Cannot have * in a filename in windows; untested as to
937 # what p4 would do in such a case.
938 if not platform.system() == "Windows":
939 path = path.replace("%2A", "*")
940 path = path.replace("%23", "#") \
941 .replace("%40", "@") \
945 def wildcard_encode(path):
946 # do % first to avoid double-encoding the %s introduced here
947 path = path.replace("%", "%25") \
948 .replace("*", "%2A") \
949 .replace("#", "%23") \
953 def wildcard_present(path):
954 m = re.search("[*#@%]", path)
957 class LargeFileSystem(object):
958 """Base class for large file system support."""
960 def __init__(self, writeToGitStream):
961 self.largeFiles = set()
962 self.writeToGitStream = writeToGitStream
964 def generatePointer(self, cloneDestination, contentFile):
965 """Return the content of a pointer file that is stored in Git instead of
966 the actual content."""
967 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
969 def pushFile(self, localLargeFile):
970 """Push the actual content which is not stored in the Git repository to
972 assert False, "Method 'pushFile' required in " + self.__class__.__name__
974 def hasLargeFileExtension(self, relPath):
977 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
981 def generateTempFile(self, contents):
982 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
986 return contentFile.name
988 def exceedsLargeFileThreshold(self, relPath, contents):
989 if gitConfigInt('git-p4.largeFileThreshold'):
990 contentsSize = sum(len(d) for d in contents)
991 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
993 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
994 contentsSize = sum(len(d) for d in contents)
995 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
997 contentTempFile = self.generateTempFile(contents)
998 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
999 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1000 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1002 compressedContentsSize = zf.infolist()[0].compress_size
1003 os.remove(contentTempFile)
1004 os.remove(compressedContentFile.name)
1005 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1009 def addLargeFile(self, relPath):
1010 self.largeFiles.add(relPath)
1012 def removeLargeFile(self, relPath):
1013 self.largeFiles.remove(relPath)
1015 def isLargeFile(self, relPath):
1016 return relPath in self.largeFiles
1018 def processContent(self, git_mode, relPath, contents):
1019 """Processes the content of git fast import. This method decides if a
1020 file is stored in the large file system and handles all necessary
1022 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1023 contentTempFile = self.generateTempFile(contents)
1024 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1025 if pointer_git_mode:
1026 git_mode = pointer_git_mode
1028 # Move temp file to final location in large file system
1029 largeFileDir = os.path.dirname(localLargeFile)
1030 if not os.path.isdir(largeFileDir):
1031 os.makedirs(largeFileDir)
1032 shutil.move(contentTempFile, localLargeFile)
1033 self.addLargeFile(relPath)
1034 if gitConfigBool('git-p4.largeFilePush'):
1035 self.pushFile(localLargeFile)
1037 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1038 return (git_mode, contents)
1040 class MockLFS(LargeFileSystem):
1041 """Mock large file system for testing."""
1043 def generatePointer(self, contentFile):
1044 """The pointer content is the original content prefixed with "pointer-".
1045 The local filename of the large file storage is derived from the file content.
1047 with open(contentFile, 'r') as f:
1050 pointerContents = 'pointer-' + content
1051 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1052 return (gitMode, pointerContents, localLargeFile)
1054 def pushFile(self, localLargeFile):
1055 """The remote filename of the large file storage is the same as the local
1056 one but in a different directory.
1058 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1059 if not os.path.exists(remotePath):
1060 os.makedirs(remotePath)
1061 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1063 class GitLFS(LargeFileSystem):
1064 """Git LFS as backend for the git-p4 large file system.
1065 See https://git-lfs.github.com/ for details."""
1067 def __init__(self, *args):
1068 LargeFileSystem.__init__(self, *args)
1069 self.baseGitAttributes = []
1071 def generatePointer(self, contentFile):
1072 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1073 mode and content which is stored in the Git repository instead of
1074 the actual content. Return also the new location of the actual
1077 if os.path.getsize(contentFile) == 0:
1078 return (None, '', None)
1080 pointerProcess = subprocess.Popen(
1081 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1082 stdout=subprocess.PIPE
1084 pointerFile = pointerProcess.stdout.read()
1085 if pointerProcess.wait():
1086 os.remove(contentFile)
1087 die('git-lfs pointer command failed. Did you install the extension?')
1089 # Git LFS removed the preamble in the output of the 'pointer' command
1090 # starting from version 1.2.0. Check for the preamble here to support
1092 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1093 if pointerFile.startswith('Git LFS pointer for'):
1094 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1096 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1097 localLargeFile = os.path.join(
1099 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1102 # LFS Spec states that pointer files should not have the executable bit set.
1104 return (gitMode, pointerFile, localLargeFile)
1106 def pushFile(self, localLargeFile):
1107 uploadProcess = subprocess.Popen(
1108 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1110 if uploadProcess.wait():
1111 die('git-lfs push command failed. Did you define a remote?')
1113 def generateGitAttributes(self):
1115 self.baseGitAttributes +
1119 '# Git LFS (see https://git-lfs.github.com/)\n',
1122 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1123 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1125 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1126 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1130 def addLargeFile(self, relPath):
1131 LargeFileSystem.addLargeFile(self, relPath)
1132 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1134 def removeLargeFile(self, relPath):
1135 LargeFileSystem.removeLargeFile(self, relPath)
1136 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1138 def processContent(self, git_mode, relPath, contents):
1139 if relPath == '.gitattributes':
1140 self.baseGitAttributes = contents
1141 return (git_mode, self.generateGitAttributes())
1143 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1147 self.usage = "usage: %prog [options]"
1148 self.needsGit = True
1149 self.verbose = False
1153 self.userMapFromPerforceServer = False
1154 self.myP4UserId = None
1158 return self.myP4UserId
1160 results = p4CmdList("user -o")
1162 if r.has_key('User'):
1163 self.myP4UserId = r['User']
1165 die("Could not find your p4 user id")
1167 def p4UserIsMe(self, p4User):
1168 # return True if the given p4 user is actually me
1169 me = self.p4UserId()
1170 if not p4User or p4User != me:
1175 def getUserCacheFilename(self):
1176 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1177 return home + "/.gitp4-usercache.txt"
1179 def getUserMapFromPerforceServer(self):
1180 if self.userMapFromPerforceServer:
1185 for output in p4CmdList("users"):
1186 if not output.has_key("User"):
1188 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1189 self.emails[output["Email"]] = output["User"]
1191 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1192 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1193 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1194 if mapUser and len(mapUser[0]) == 3:
1195 user = mapUser[0][0]
1196 fullname = mapUser[0][1]
1197 email = mapUser[0][2]
1198 self.users[user] = fullname + " <" + email + ">"
1199 self.emails[email] = user
1202 for (key, val) in self.users.items():
1203 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1205 open(self.getUserCacheFilename(), "wb").write(s)
1206 self.userMapFromPerforceServer = True
1208 def loadUserMapFromCache(self):
1210 self.userMapFromPerforceServer = False
1212 cache = open(self.getUserCacheFilename(), "rb")
1213 lines = cache.readlines()
1216 entry = line.strip().split("\t")
1217 self.users[entry[0]] = entry[1]
1219 self.getUserMapFromPerforceServer()
1221 class P4Debug(Command):
1223 Command.__init__(self)
1225 self.description = "A tool to debug the output of p4 -G."
1226 self.needsGit = False
1228 def run(self, args):
1230 for output in p4CmdList(args):
1231 print 'Element: %d' % j
1236 class P4RollBack(Command):
1238 Command.__init__(self)
1240 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1242 self.description = "A tool to debug the multi-branch import. Don't use :)"
1243 self.rollbackLocalBranches = False
1245 def run(self, args):
1248 maxChange = int(args[0])
1250 if "p4ExitCode" in p4Cmd("changes -m 1"):
1251 die("Problems executing p4");
1253 if self.rollbackLocalBranches:
1254 refPrefix = "refs/heads/"
1255 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1257 refPrefix = "refs/remotes/"
1258 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1261 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1263 ref = refPrefix + line
1264 log = extractLogMessageFromGitCommit(ref)
1265 settings = extractSettingsGitLog(log)
1267 depotPaths = settings['depot-paths']
1268 change = settings['change']
1272 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1273 for p in depotPaths]))) == 0:
1274 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1275 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1278 while change and int(change) > maxChange:
1281 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1282 system("git update-ref %s \"%s^\"" % (ref, ref))
1283 log = extractLogMessageFromGitCommit(ref)
1284 settings = extractSettingsGitLog(log)
1287 depotPaths = settings['depot-paths']
1288 change = settings['change']
1291 print "%s rewound to %s" % (ref, change)
1295 class P4Submit(Command, P4UserMap):
1297 conflict_behavior_choices = ("ask", "skip", "quit")
1300 Command.__init__(self)
1301 P4UserMap.__init__(self)
1303 optparse.make_option("--origin", dest="origin"),
1304 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1305 # preserve the user, requires relevant p4 permissions
1306 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1307 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1308 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1309 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1310 optparse.make_option("--conflict", dest="conflict_behavior",
1311 choices=self.conflict_behavior_choices),
1312 optparse.make_option("--branch", dest="branch"),
1313 optparse.make_option("--shelve", dest="shelve", action="store_true",
1314 help="Shelve instead of submit. Shelved files are reverted, "
1315 "restoring the workspace to the state before the shelve"),
1316 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1317 metavar="CHANGELIST",
1318 help="update an existing shelved changelist, implies --shelve")
1320 self.description = "Submit changes from git to the perforce depot."
1321 self.usage += " [name of git branch to submit into perforce depot]"
1323 self.detectRenames = False
1324 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1325 self.dry_run = False
1327 self.update_shelve = None
1328 self.prepare_p4_only = False
1329 self.conflict_behavior = None
1330 self.isWindows = (platform.system() == "Windows")
1331 self.exportLabels = False
1332 self.p4HasMoveCommand = p4_has_move_command()
1335 if gitConfig('git-p4.largeFileSystem'):
1336 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1339 if len(p4CmdList("opened ...")) > 0:
1340 die("You have files opened with perforce! Close them before starting the sync.")
1342 def separate_jobs_from_description(self, message):
1343 """Extract and return a possible Jobs field in the commit
1344 message. It goes into a separate section in the p4 change
1347 A jobs line starts with "Jobs:" and looks like a new field
1348 in a form. Values are white-space separated on the same
1349 line or on following lines that start with a tab.
1351 This does not parse and extract the full git commit message
1352 like a p4 form. It just sees the Jobs: line as a marker
1353 to pass everything from then on directly into the p4 form,
1354 but outside the description section.
1356 Return a tuple (stripped log message, jobs string)."""
1358 m = re.search(r'^Jobs:', message, re.MULTILINE)
1360 return (message, None)
1362 jobtext = message[m.start():]
1363 stripped_message = message[:m.start()].rstrip()
1364 return (stripped_message, jobtext)
1366 def prepareLogMessage(self, template, message, jobs):
1367 """Edits the template returned from "p4 change -o" to insert
1368 the message in the Description field, and the jobs text in
1372 inDescriptionSection = False
1374 for line in template.split("\n"):
1375 if line.startswith("#"):
1376 result += line + "\n"
1379 if inDescriptionSection:
1380 if line.startswith("Files:") or line.startswith("Jobs:"):
1381 inDescriptionSection = False
1382 # insert Jobs section
1384 result += jobs + "\n"
1388 if line.startswith("Description:"):
1389 inDescriptionSection = True
1391 for messageLine in message.split("\n"):
1392 line += "\t" + messageLine + "\n"
1394 result += line + "\n"
1398 def patchRCSKeywords(self, file, pattern):
1399 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1400 (handle, outFileName) = tempfile.mkstemp(dir='.')
1402 outFile = os.fdopen(handle, "w+")
1403 inFile = open(file, "r")
1404 regexp = re.compile(pattern, re.VERBOSE)
1405 for line in inFile.readlines():
1406 line = regexp.sub(r'$\1$', line)
1410 # Forcibly overwrite the original file
1412 shutil.move(outFileName, file)
1414 # cleanup our temporary file
1415 os.unlink(outFileName)
1416 print "Failed to strip RCS keywords in %s" % file
1419 print "Patched up RCS keywords in %s" % file
1421 def p4UserForCommit(self,id):
1422 # Return the tuple (perforce user,git email) for a given git commit id
1423 self.getUserMapFromPerforceServer()
1424 gitEmail = read_pipe(["git", "log", "--max-count=1",
1425 "--format=%ae", id])
1426 gitEmail = gitEmail.strip()
1427 if not self.emails.has_key(gitEmail):
1428 return (None,gitEmail)
1430 return (self.emails[gitEmail],gitEmail)
1432 def checkValidP4Users(self,commits):
1433 # check if any git authors cannot be mapped to p4 users
1435 (user,email) = self.p4UserForCommit(id)
1437 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1438 if gitConfigBool("git-p4.allowMissingP4Users"):
1441 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1443 def lastP4Changelist(self):
1444 # Get back the last changelist number submitted in this client spec. This
1445 # then gets used to patch up the username in the change. If the same
1446 # client spec is being used by multiple processes then this might go
1448 results = p4CmdList("client -o") # find the current client
1451 if r.has_key('Client'):
1452 client = r['Client']
1455 die("could not get client spec")
1456 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1458 if r.has_key('change'):
1460 die("Could not get changelist number for last submit - cannot patch up user details")
1462 def modifyChangelistUser(self, changelist, newUser):
1463 # fixup the user field of a changelist after it has been submitted.
1464 changes = p4CmdList("change -o %s" % changelist)
1465 if len(changes) != 1:
1466 die("Bad output from p4 change modifying %s to user %s" %
1467 (changelist, newUser))
1470 if c['User'] == newUser: return # nothing to do
1472 input = marshal.dumps(c)
1474 result = p4CmdList("change -f -i", stdin=input)
1476 if r.has_key('code'):
1477 if r['code'] == 'error':
1478 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1479 if r.has_key('data'):
1480 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1482 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1484 def canChangeChangelists(self):
1485 # check to see if we have p4 admin or super-user permissions, either of
1486 # which are required to modify changelists.
1487 results = p4CmdList(["protects", self.depotPath])
1489 if r.has_key('perm'):
1490 if r['perm'] == 'admin':
1492 if r['perm'] == 'super':
1496 def prepareSubmitTemplate(self, changelist=None):
1497 """Run "p4 change -o" to grab a change specification template.
1498 This does not use "p4 -G", as it is nice to keep the submission
1499 template in original order, since a human might edit it.
1501 Remove lines in the Files section that show changes to files
1502 outside the depot path we're committing into."""
1504 [upstream, settings] = findUpstreamBranchPoint()
1507 inFilesSection = False
1508 args = ['change', '-o']
1510 args.append(str(changelist))
1512 for line in p4_read_pipe_lines(args):
1513 if line.endswith("\r\n"):
1514 line = line[:-2] + "\n"
1516 if line.startswith("\t"):
1517 # path starts and ends with a tab
1519 lastTab = path.rfind("\t")
1521 path = path[:lastTab]
1522 if settings.has_key('depot-paths'):
1523 if not [p for p in settings['depot-paths']
1524 if p4PathStartsWith(path, p)]:
1527 if not p4PathStartsWith(path, self.depotPath):
1530 inFilesSection = False
1532 if line.startswith("Files:"):
1533 inFilesSection = True
1539 def edit_template(self, template_file):
1540 """Invoke the editor to let the user change the submission
1541 message. Return true if okay to continue with the submit."""
1543 # if configured to skip the editing part, just submit
1544 if gitConfigBool("git-p4.skipSubmitEdit"):
1547 # look at the modification time, to check later if the user saved
1549 mtime = os.stat(template_file).st_mtime
1552 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1553 editor = os.environ.get("P4EDITOR")
1555 editor = read_pipe("git var GIT_EDITOR").strip()
1556 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1558 # If the file was not saved, prompt to see if this patch should
1559 # be skipped. But skip this verification step if configured so.
1560 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1563 # modification time updated means user saved the file
1564 if os.stat(template_file).st_mtime > mtime:
1568 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1574 def get_diff_description(self, editedFiles, filesToAdd):
1576 if os.environ.has_key("P4DIFF"):
1577 del(os.environ["P4DIFF"])
1579 for editedFile in editedFiles:
1580 diff += p4_read_pipe(['diff', '-du',
1581 wildcard_encode(editedFile)])
1585 for newFile in filesToAdd:
1586 newdiff += "==== new file ====\n"
1587 newdiff += "--- /dev/null\n"
1588 newdiff += "+++ %s\n" % newFile
1589 f = open(newFile, "r")
1590 for line in f.readlines():
1591 newdiff += "+" + line
1594 return (diff + newdiff).replace('\r\n', '\n')
1596 def applyCommit(self, id):
1597 """Apply one commit, return True if it succeeded."""
1599 print "Applying", read_pipe(["git", "show", "-s",
1600 "--format=format:%h %s", id])
1602 (p4User, gitEmail) = self.p4UserForCommit(id)
1604 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1606 filesToChangeType = set()
1607 filesToDelete = set()
1609 pureRenameCopy = set()
1610 filesToChangeExecBit = {}
1614 diff = parseDiffTreeEntry(line)
1615 modifier = diff['status']
1617 all_files.append(path)
1621 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1622 filesToChangeExecBit[path] = diff['dst_mode']
1623 editedFiles.add(path)
1624 elif modifier == "A":
1625 filesToAdd.add(path)
1626 filesToChangeExecBit[path] = diff['dst_mode']
1627 if path in filesToDelete:
1628 filesToDelete.remove(path)
1629 elif modifier == "D":
1630 filesToDelete.add(path)
1631 if path in filesToAdd:
1632 filesToAdd.remove(path)
1633 elif modifier == "C":
1634 src, dest = diff['src'], diff['dst']
1635 p4_integrate(src, dest)
1636 pureRenameCopy.add(dest)
1637 if diff['src_sha1'] != diff['dst_sha1']:
1639 pureRenameCopy.discard(dest)
1640 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1642 pureRenameCopy.discard(dest)
1643 filesToChangeExecBit[dest] = diff['dst_mode']
1645 # turn off read-only attribute
1646 os.chmod(dest, stat.S_IWRITE)
1648 editedFiles.add(dest)
1649 elif modifier == "R":
1650 src, dest = diff['src'], diff['dst']
1651 if self.p4HasMoveCommand:
1652 p4_edit(src) # src must be open before move
1653 p4_move(src, dest) # opens for (move/delete, move/add)
1655 p4_integrate(src, dest)
1656 if diff['src_sha1'] != diff['dst_sha1']:
1659 pureRenameCopy.add(dest)
1660 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1661 if not self.p4HasMoveCommand:
1662 p4_edit(dest) # with move: already open, writable
1663 filesToChangeExecBit[dest] = diff['dst_mode']
1664 if not self.p4HasMoveCommand:
1666 os.chmod(dest, stat.S_IWRITE)
1668 filesToDelete.add(src)
1669 editedFiles.add(dest)
1670 elif modifier == "T":
1671 filesToChangeType.add(path)
1673 die("unknown modifier %s for %s" % (modifier, path))
1675 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1676 patchcmd = diffcmd + " | git apply "
1677 tryPatchCmd = patchcmd + "--check -"
1678 applyPatchCmd = patchcmd + "--check --apply -"
1679 patch_succeeded = True
1681 if os.system(tryPatchCmd) != 0:
1682 fixed_rcs_keywords = False
1683 patch_succeeded = False
1684 print "Unfortunately applying the change failed!"
1686 # Patch failed, maybe it's just RCS keyword woes. Look through
1687 # the patch to see if that's possible.
1688 if gitConfigBool("git-p4.attemptRCSCleanup"):
1692 for file in editedFiles | filesToDelete:
1693 # did this file's delta contain RCS keywords?
1694 pattern = p4_keywords_regexp_for_file(file)
1697 # this file is a possibility...look for RCS keywords.
1698 regexp = re.compile(pattern, re.VERBOSE)
1699 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1700 if regexp.search(line):
1702 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1703 kwfiles[file] = pattern
1706 for file in kwfiles:
1708 print "zapping %s with %s" % (line,pattern)
1709 # File is being deleted, so not open in p4. Must
1710 # disable the read-only bit on windows.
1711 if self.isWindows and file not in editedFiles:
1712 os.chmod(file, stat.S_IWRITE)
1713 self.patchRCSKeywords(file, kwfiles[file])
1714 fixed_rcs_keywords = True
1716 if fixed_rcs_keywords:
1717 print "Retrying the patch with RCS keywords cleaned up"
1718 if os.system(tryPatchCmd) == 0:
1719 patch_succeeded = True
1721 if not patch_succeeded:
1722 for f in editedFiles:
1727 # Apply the patch for real, and do add/delete/+x handling.
1729 system(applyPatchCmd)
1731 for f in filesToChangeType:
1732 p4_edit(f, "-t", "auto")
1733 for f in filesToAdd:
1735 for f in filesToDelete:
1739 # Set/clear executable bits
1740 for f in filesToChangeExecBit.keys():
1741 mode = filesToChangeExecBit[f]
1742 setP4ExecBit(f, mode)
1744 if self.update_shelve:
1745 print("all_files = %s" % str(all_files))
1746 p4_reopen_in_change(self.update_shelve, all_files)
1749 # Build p4 change description, starting with the contents
1750 # of the git commit message.
1752 logMessage = extractLogMessageFromGitCommit(id)
1753 logMessage = logMessage.strip()
1754 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1756 template = self.prepareSubmitTemplate(self.update_shelve)
1757 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1759 if self.preserveUser:
1760 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1762 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1763 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1764 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1765 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1767 separatorLine = "######## everything below this line is just the diff #######\n"
1768 if not self.prepare_p4_only:
1769 submitTemplate += separatorLine
1770 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1772 (handle, fileName) = tempfile.mkstemp()
1773 tmpFile = os.fdopen(handle, "w+b")
1775 submitTemplate = submitTemplate.replace("\n", "\r\n")
1776 tmpFile.write(submitTemplate)
1779 if self.prepare_p4_only:
1781 # Leave the p4 tree prepared, and the submit template around
1782 # and let the user decide what to do next
1785 print "P4 workspace prepared for submission."
1786 print "To submit or revert, go to client workspace"
1787 print " " + self.clientPath
1789 print "To submit, use \"p4 submit\" to write a new description,"
1790 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1791 " \"git p4\"." % fileName
1792 print "You can delete the file \"%s\" when finished." % fileName
1794 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1795 print "To preserve change ownership by user %s, you must\n" \
1796 "do \"p4 change -f <change>\" after submitting and\n" \
1797 "edit the User field."
1799 print "After submitting, renamed files must be re-synced."
1800 print "Invoke \"p4 sync -f\" on each of these files:"
1801 for f in pureRenameCopy:
1805 print "To revert the changes, use \"p4 revert ...\", and delete"
1806 print "the submit template file \"%s\"" % fileName
1808 print "Since the commit adds new files, they must be deleted:"
1809 for f in filesToAdd:
1815 # Let the user edit the change description, then submit it.
1820 if self.edit_template(fileName):
1821 # read the edited message and submit
1822 tmpFile = open(fileName, "rb")
1823 message = tmpFile.read()
1826 message = message.replace("\r\n", "\n")
1827 submitTemplate = message[:message.index(separatorLine)]
1829 if self.update_shelve:
1830 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1832 p4_write_pipe(['shelve', '-i'], submitTemplate)
1834 p4_write_pipe(['submit', '-i'], submitTemplate)
1835 # The rename/copy happened by applying a patch that created a
1836 # new file. This leaves it writable, which confuses p4.
1837 for f in pureRenameCopy:
1840 if self.preserveUser:
1842 # Get last changelist number. Cannot easily get it from
1843 # the submit command output as the output is
1845 changelist = self.lastP4Changelist()
1846 self.modifyChangelistUser(changelist, p4User)
1852 if not submitted or self.shelve:
1854 print ("Reverting shelved files.")
1856 print ("Submission cancelled, undoing p4 changes.")
1857 for f in editedFiles | filesToDelete:
1859 for f in filesToAdd:
1866 # Export git tags as p4 labels. Create a p4 label and then tag
1868 def exportGitTags(self, gitTags):
1869 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1870 if len(validLabelRegexp) == 0:
1871 validLabelRegexp = defaultLabelRegexp
1872 m = re.compile(validLabelRegexp)
1874 for name in gitTags:
1876 if not m.match(name):
1878 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1881 # Get the p4 commit this corresponds to
1882 logMessage = extractLogMessageFromGitCommit(name)
1883 values = extractSettingsGitLog(logMessage)
1885 if not values.has_key('change'):
1886 # a tag pointing to something not sent to p4; ignore
1888 print "git tag %s does not give a p4 commit" % name
1891 changelist = values['change']
1893 # Get the tag details.
1897 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1900 if re.match(r'tag\s+', l):
1902 elif re.match(r'\s*$', l):
1909 body = ["lightweight tag imported by git p4\n"]
1911 # Create the label - use the same view as the client spec we are using
1912 clientSpec = getClientSpec()
1914 labelTemplate = "Label: %s\n" % name
1915 labelTemplate += "Description:\n"
1917 labelTemplate += "\t" + b + "\n"
1918 labelTemplate += "View:\n"
1919 for depot_side in clientSpec.mappings:
1920 labelTemplate += "\t%s\n" % depot_side
1923 print "Would create p4 label %s for tag" % name
1924 elif self.prepare_p4_only:
1925 print "Not creating p4 label %s for tag due to option" \
1926 " --prepare-p4-only" % name
1928 p4_write_pipe(["label", "-i"], labelTemplate)
1931 p4_system(["tag", "-l", name] +
1932 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1935 print "created p4 label for tag %s" % name
1937 def run(self, args):
1939 self.master = currentGitBranch()
1940 elif len(args) == 1:
1941 self.master = args[0]
1942 if not branchExists(self.master):
1943 die("Branch %s does not exist" % self.master)
1948 allowSubmit = gitConfig("git-p4.allowSubmit")
1949 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1950 die("%s is not in git-p4.allowSubmit" % self.master)
1952 [upstream, settings] = findUpstreamBranchPoint()
1953 self.depotPath = settings['depot-paths'][0]
1954 if len(self.origin) == 0:
1955 self.origin = upstream
1957 if self.update_shelve:
1960 if self.preserveUser:
1961 if not self.canChangeChangelists():
1962 die("Cannot preserve user names without p4 super-user or admin permissions")
1964 # if not set from the command line, try the config file
1965 if self.conflict_behavior is None:
1966 val = gitConfig("git-p4.conflict")
1968 if val not in self.conflict_behavior_choices:
1969 die("Invalid value '%s' for config git-p4.conflict" % val)
1972 self.conflict_behavior = val
1975 print "Origin branch is " + self.origin
1977 if len(self.depotPath) == 0:
1978 print "Internal error: cannot locate perforce depot path from existing branches"
1981 self.useClientSpec = False
1982 if gitConfigBool("git-p4.useclientspec"):
1983 self.useClientSpec = True
1984 if self.useClientSpec:
1985 self.clientSpecDirs = getClientSpec()
1987 # Check for the existence of P4 branches
1988 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1990 if self.useClientSpec and not branchesDetected:
1991 # all files are relative to the client spec
1992 self.clientPath = getClientRoot()
1994 self.clientPath = p4Where(self.depotPath)
1996 if self.clientPath == "":
1997 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1999 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2000 self.oldWorkingDirectory = os.getcwd()
2002 # ensure the clientPath exists
2003 new_client_dir = False
2004 if not os.path.exists(self.clientPath):
2005 new_client_dir = True
2006 os.makedirs(self.clientPath)
2008 chdir(self.clientPath, is_client_path=True)
2010 print "Would synchronize p4 checkout in %s" % self.clientPath
2012 print "Synchronizing p4 checkout..."
2014 # old one was destroyed, and maybe nobody told p4
2015 p4_sync("...", "-f")
2022 commitish = self.master
2026 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2027 commits.append(line.strip())
2030 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2031 self.checkAuthorship = False
2033 self.checkAuthorship = True
2035 if self.preserveUser:
2036 self.checkValidP4Users(commits)
2039 # Build up a set of options to be passed to diff when
2040 # submitting each commit to p4.
2042 if self.detectRenames:
2043 # command-line -M arg
2044 self.diffOpts = "-M"
2046 # If not explicitly set check the config variable
2047 detectRenames = gitConfig("git-p4.detectRenames")
2049 if detectRenames.lower() == "false" or detectRenames == "":
2051 elif detectRenames.lower() == "true":
2052 self.diffOpts = "-M"
2054 self.diffOpts = "-M%s" % detectRenames
2056 # no command-line arg for -C or --find-copies-harder, just
2058 detectCopies = gitConfig("git-p4.detectCopies")
2059 if detectCopies.lower() == "false" or detectCopies == "":
2061 elif detectCopies.lower() == "true":
2062 self.diffOpts += " -C"
2064 self.diffOpts += " -C%s" % detectCopies
2066 if gitConfigBool("git-p4.detectCopiesHarder"):
2067 self.diffOpts += " --find-copies-harder"
2070 # Apply the commits, one at a time. On failure, ask if should
2071 # continue to try the rest of the patches, or quit.
2076 last = len(commits) - 1
2077 for i, commit in enumerate(commits):
2079 print " ", read_pipe(["git", "show", "-s",
2080 "--format=format:%h %s", commit])
2083 ok = self.applyCommit(commit)
2085 applied.append(commit)
2087 if self.prepare_p4_only and i < last:
2088 print "Processing only the first commit due to option" \
2089 " --prepare-p4-only"
2094 # prompt for what to do, or use the option/variable
2095 if self.conflict_behavior == "ask":
2096 print "What do you want to do?"
2097 response = raw_input("[s]kip this commit but apply"
2098 " the rest, or [q]uit? ")
2101 elif self.conflict_behavior == "skip":
2103 elif self.conflict_behavior == "quit":
2106 die("Unknown conflict_behavior '%s'" %
2107 self.conflict_behavior)
2109 if response[0] == "s":
2110 print "Skipping this commit, but applying the rest"
2112 if response[0] == "q":
2119 chdir(self.oldWorkingDirectory)
2120 shelved_applied = "shelved" if self.shelve else "applied"
2123 elif self.prepare_p4_only:
2125 elif len(commits) == len(applied):
2126 print ("All commits {0}!".format(shelved_applied))
2130 sync.branch = self.branch
2137 if len(applied) == 0:
2138 print ("No commits {0}.".format(shelved_applied))
2140 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2146 print star, read_pipe(["git", "show", "-s",
2147 "--format=format:%h %s", c])
2148 print "You will have to do 'git p4 sync' and rebase."
2150 if gitConfigBool("git-p4.exportLabels"):
2151 self.exportLabels = True
2153 if self.exportLabels:
2154 p4Labels = getP4Labels(self.depotPath)
2155 gitTags = getGitTags()
2157 missingGitTags = gitTags - p4Labels
2158 self.exportGitTags(missingGitTags)
2160 # exit with error unless everything applied perfectly
2161 if len(commits) != len(applied):
2167 """Represent a p4 view ("p4 help views"), and map files in a
2168 repo according to the view."""
2170 def __init__(self, client_name):
2172 self.client_prefix = "//%s/" % client_name
2173 # cache results of "p4 where" to lookup client file locations
2174 self.client_spec_path_cache = {}
2176 def append(self, view_line):
2177 """Parse a view line, splitting it into depot and client
2178 sides. Append to self.mappings, preserving order. This
2179 is only needed for tag creation."""
2181 # Split the view line into exactly two words. P4 enforces
2182 # structure on these lines that simplifies this quite a bit.
2184 # Either or both words may be double-quoted.
2185 # Single quotes do not matter.
2186 # Double-quote marks cannot occur inside the words.
2187 # A + or - prefix is also inside the quotes.
2188 # There are no quotes unless they contain a space.
2189 # The line is already white-space stripped.
2190 # The two words are separated by a single space.
2192 if view_line[0] == '"':
2193 # First word is double quoted. Find its end.
2194 close_quote_index = view_line.find('"', 1)
2195 if close_quote_index <= 0:
2196 die("No first-word closing quote found: %s" % view_line)
2197 depot_side = view_line[1:close_quote_index]
2198 # skip closing quote and space
2199 rhs_index = close_quote_index + 1 + 1
2201 space_index = view_line.find(" ")
2202 if space_index <= 0:
2203 die("No word-splitting space found: %s" % view_line)
2204 depot_side = view_line[0:space_index]
2205 rhs_index = space_index + 1
2207 # prefix + means overlay on previous mapping
2208 if depot_side.startswith("+"):
2209 depot_side = depot_side[1:]
2211 # prefix - means exclude this path, leave out of mappings
2213 if depot_side.startswith("-"):
2215 depot_side = depot_side[1:]
2218 self.mappings.append(depot_side)
2220 def convert_client_path(self, clientFile):
2221 # chop off //client/ part to make it relative
2222 if not clientFile.startswith(self.client_prefix):
2223 die("No prefix '%s' on clientFile '%s'" %
2224 (self.client_prefix, clientFile))
2225 return clientFile[len(self.client_prefix):]
2227 def update_client_spec_path_cache(self, files):
2228 """ Caching file paths by "p4 where" batch query """
2230 # List depot file paths exclude that already cached
2231 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2233 if len(fileArgs) == 0:
2234 return # All files in cache
2236 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2237 for res in where_result:
2238 if "code" in res and res["code"] == "error":
2239 # assume error is "... file(s) not in client view"
2241 if "clientFile" not in res:
2242 die("No clientFile in 'p4 where' output")
2244 # it will list all of them, but only one not unmap-ped
2246 if gitConfigBool("core.ignorecase"):
2247 res['depotFile'] = res['depotFile'].lower()
2248 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2250 # not found files or unmap files set to ""
2251 for depotFile in fileArgs:
2252 if gitConfigBool("core.ignorecase"):
2253 depotFile = depotFile.lower()
2254 if depotFile not in self.client_spec_path_cache:
2255 self.client_spec_path_cache[depotFile] = ""
2257 def map_in_client(self, depot_path):
2258 """Return the relative location in the client where this
2259 depot file should live. Returns "" if the file should
2260 not be mapped in the client."""
2262 if gitConfigBool("core.ignorecase"):
2263 depot_path = depot_path.lower()
2265 if depot_path in self.client_spec_path_cache:
2266 return self.client_spec_path_cache[depot_path]
2268 die( "Error: %s is not found in client spec path" % depot_path )
2271 class P4Sync(Command, P4UserMap):
2272 delete_actions = ( "delete", "move/delete", "purge" )
2275 Command.__init__(self)
2276 P4UserMap.__init__(self)
2278 optparse.make_option("--branch", dest="branch"),
2279 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2280 optparse.make_option("--changesfile", dest="changesFile"),
2281 optparse.make_option("--silent", dest="silent", action="store_true"),
2282 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2283 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2284 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2285 help="Import into refs/heads/ , not refs/remotes"),
2286 optparse.make_option("--max-changes", dest="maxChanges",
2287 help="Maximum number of changes to import"),
2288 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2289 help="Internal block size to use when iteratively calling p4 changes"),
2290 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2291 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2292 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2293 help="Only sync files that are included in the Perforce Client Spec"),
2294 optparse.make_option("-/", dest="cloneExclude",
2295 action="append", type="string",
2296 help="exclude depot path"),
2298 self.description = """Imports from Perforce into a git repository.\n
2300 //depot/my/project/ -- to import the current head
2301 //depot/my/project/@all -- to import everything
2302 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2304 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2306 self.usage += " //depot/path[@revRange]"
2308 self.createdBranches = set()
2309 self.committedChanges = set()
2311 self.detectBranches = False
2312 self.detectLabels = False
2313 self.importLabels = False
2314 self.changesFile = ""
2315 self.syncWithOrigin = True
2316 self.importIntoRemotes = True
2317 self.maxChanges = ""
2318 self.changes_block_size = None
2319 self.keepRepoPath = False
2320 self.depotPaths = None
2321 self.p4BranchesInGit = []
2322 self.cloneExclude = []
2323 self.useClientSpec = False
2324 self.useClientSpec_from_options = False
2325 self.clientSpecDirs = None
2326 self.tempBranches = []
2327 self.tempBranchLocation = "refs/git-p4-tmp"
2328 self.largeFileSystem = None
2330 if gitConfig('git-p4.largeFileSystem'):
2331 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2332 self.largeFileSystem = largeFileSystemConstructor(
2333 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2336 if gitConfig("git-p4.syncFromOrigin") == "false":
2337 self.syncWithOrigin = False
2339 # This is required for the "append" cloneExclude action
2340 def ensure_value(self, attr, value):
2341 if not hasattr(self, attr) or getattr(self, attr) is None:
2342 setattr(self, attr, value)
2343 return getattr(self, attr)
2345 # Force a checkpoint in fast-import and wait for it to finish
2346 def checkpoint(self):
2347 self.gitStream.write("checkpoint\n\n")
2348 self.gitStream.write("progress checkpoint\n\n")
2349 out = self.gitOutput.readline()
2351 print "checkpoint finished: " + out
2353 def extractFilesFromCommit(self, commit):
2354 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2355 for path in self.cloneExclude]
2358 while commit.has_key("depotFile%s" % fnum):
2359 path = commit["depotFile%s" % fnum]
2361 if [p for p in self.cloneExclude
2362 if p4PathStartsWith(path, p)]:
2365 found = [p for p in self.depotPaths
2366 if p4PathStartsWith(path, p)]
2373 file["rev"] = commit["rev%s" % fnum]
2374 file["action"] = commit["action%s" % fnum]
2375 file["type"] = commit["type%s" % fnum]
2380 def extractJobsFromCommit(self, commit):
2383 while commit.has_key("job%s" % jnum):
2384 job = commit["job%s" % jnum]
2389 def stripRepoPath(self, path, prefixes):
2390 """When streaming files, this is called to map a p4 depot path
2391 to where it should go in git. The prefixes are either
2392 self.depotPaths, or self.branchPrefixes in the case of
2393 branch detection."""
2395 if self.useClientSpec:
2396 # branch detection moves files up a level (the branch name)
2397 # from what client spec interpretation gives
2398 path = self.clientSpecDirs.map_in_client(path)
2399 if self.detectBranches:
2400 for b in self.knownBranches:
2401 if path.startswith(b + "/"):
2402 path = path[len(b)+1:]
2404 elif self.keepRepoPath:
2405 # Preserve everything in relative path name except leading
2406 # //depot/; just look at first prefix as they all should
2407 # be in the same depot.
2408 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2409 if p4PathStartsWith(path, depot):
2410 path = path[len(depot):]
2414 if p4PathStartsWith(path, p):
2415 path = path[len(p):]
2418 path = wildcard_decode(path)
2421 def splitFilesIntoBranches(self, commit):
2422 """Look at each depotFile in the commit to figure out to what
2423 branch it belongs."""
2425 if self.clientSpecDirs:
2426 files = self.extractFilesFromCommit(commit)
2427 self.clientSpecDirs.update_client_spec_path_cache(files)
2431 while commit.has_key("depotFile%s" % fnum):
2432 path = commit["depotFile%s" % fnum]
2433 found = [p for p in self.depotPaths
2434 if p4PathStartsWith(path, p)]
2441 file["rev"] = commit["rev%s" % fnum]
2442 file["action"] = commit["action%s" % fnum]
2443 file["type"] = commit["type%s" % fnum]
2446 # start with the full relative path where this file would
2448 if self.useClientSpec:
2449 relPath = self.clientSpecDirs.map_in_client(path)
2451 relPath = self.stripRepoPath(path, self.depotPaths)
2453 for branch in self.knownBranches.keys():
2454 # add a trailing slash so that a commit into qt/4.2foo
2455 # doesn't end up in qt/4.2, e.g.
2456 if relPath.startswith(branch + "/"):
2457 if branch not in branches:
2458 branches[branch] = []
2459 branches[branch].append(file)
2464 def writeToGitStream(self, gitMode, relPath, contents):
2465 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2466 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2468 self.gitStream.write(d)
2469 self.gitStream.write('\n')
2471 # output one file from the P4 stream
2472 # - helper for streamP4Files
2474 def streamOneP4File(self, file, contents):
2475 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2477 size = int(self.stream_file['fileSize'])
2478 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2481 (type_base, type_mods) = split_p4_type(file["type"])
2484 if "x" in type_mods:
2486 if type_base == "symlink":
2488 # p4 print on a symlink sometimes contains "target\n";
2489 # if it does, remove the newline
2490 data = ''.join(contents)
2492 # Some version of p4 allowed creating a symlink that pointed
2493 # to nothing. This causes p4 errors when checking out such
2494 # a change, and errors here too. Work around it by ignoring
2495 # the bad symlink; hopefully a future change fixes it.
2496 print "\nIgnoring empty symlink in %s" % file['depotFile']
2498 elif data[-1] == '\n':
2499 contents = [data[:-1]]
2503 if type_base == "utf16":
2504 # p4 delivers different text in the python output to -G
2505 # than it does when using "print -o", or normal p4 client
2506 # operations. utf16 is converted to ascii or utf8, perhaps.
2507 # But ascii text saved as -t utf16 is completely mangled.
2508 # Invoke print -o to get the real contents.
2510 # On windows, the newlines will always be mangled by print, so put
2511 # them back too. This is not needed to the cygwin windows version,
2512 # just the native "NT" type.
2515 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2516 except Exception as e:
2517 if 'Translation of file content failed' in str(e):
2518 type_base = 'binary'
2522 if p4_version_string().find('/NT') >= 0:
2523 text = text.replace('\r\n', '\n')
2526 if type_base == "apple":
2527 # Apple filetype files will be streamed as a concatenation of
2528 # its appledouble header and the contents. This is useless
2529 # on both macs and non-macs. If using "print -q -o xx", it
2530 # will create "xx" with the data, and "%xx" with the header.
2531 # This is also not very useful.
2533 # Ideally, someday, this script can learn how to generate
2534 # appledouble files directly and import those to git, but
2535 # non-mac machines can never find a use for apple filetype.
2536 print "\nIgnoring apple filetype file %s" % file['depotFile']
2539 # Note that we do not try to de-mangle keywords on utf16 files,
2540 # even though in theory somebody may want that.
2541 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2543 regexp = re.compile(pattern, re.VERBOSE)
2544 text = ''.join(contents)
2545 text = regexp.sub(r'$\1$', text)
2549 relPath.decode('ascii')
2552 if gitConfig('git-p4.pathEncoding'):
2553 encoding = gitConfig('git-p4.pathEncoding')
2554 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2556 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2558 if self.largeFileSystem:
2559 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2561 self.writeToGitStream(git_mode, relPath, contents)
2563 def streamOneP4Deletion(self, file):
2564 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2566 sys.stdout.write("delete %s\n" % relPath)
2568 self.gitStream.write("D %s\n" % relPath)
2570 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2571 self.largeFileSystem.removeLargeFile(relPath)
2573 # handle another chunk of streaming data
2574 def streamP4FilesCb(self, marshalled):
2576 # catch p4 errors and complain
2578 if "code" in marshalled:
2579 if marshalled["code"] == "error":
2580 if "data" in marshalled:
2581 err = marshalled["data"].rstrip()
2583 if not err and 'fileSize' in self.stream_file:
2584 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2585 if required_bytes > 0:
2586 err = 'Not enough space left on %s! Free at least %i MB.' % (
2587 os.getcwd(), required_bytes/1024/1024
2592 if self.stream_have_file_info:
2593 if "depotFile" in self.stream_file:
2594 f = self.stream_file["depotFile"]
2595 # force a failure in fast-import, else an empty
2596 # commit will be made
2597 self.gitStream.write("\n")
2598 self.gitStream.write("die-now\n")
2599 self.gitStream.close()
2600 # ignore errors, but make sure it exits first
2601 self.importProcess.wait()
2603 die("Error from p4 print for %s: %s" % (f, err))
2605 die("Error from p4 print: %s" % err)
2607 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2608 # start of a new file - output the old one first
2609 self.streamOneP4File(self.stream_file, self.stream_contents)
2610 self.stream_file = {}
2611 self.stream_contents = []
2612 self.stream_have_file_info = False
2614 # pick up the new file information... for the
2615 # 'data' field we need to append to our array
2616 for k in marshalled.keys():
2618 if 'streamContentSize' not in self.stream_file:
2619 self.stream_file['streamContentSize'] = 0
2620 self.stream_file['streamContentSize'] += len(marshalled['data'])
2621 self.stream_contents.append(marshalled['data'])
2623 self.stream_file[k] = marshalled[k]
2626 'streamContentSize' in self.stream_file and
2627 'fileSize' in self.stream_file and
2628 'depotFile' in self.stream_file):
2629 size = int(self.stream_file["fileSize"])
2631 progress = 100*self.stream_file['streamContentSize']/size
2632 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2635 self.stream_have_file_info = True
2637 # Stream directly from "p4 files" into "git fast-import"
2638 def streamP4Files(self, files):
2644 filesForCommit.append(f)
2645 if f['action'] in self.delete_actions:
2646 filesToDelete.append(f)
2648 filesToRead.append(f)
2651 for f in filesToDelete:
2652 self.streamOneP4Deletion(f)
2654 if len(filesToRead) > 0:
2655 self.stream_file = {}
2656 self.stream_contents = []
2657 self.stream_have_file_info = False
2659 # curry self argument
2660 def streamP4FilesCbSelf(entry):
2661 self.streamP4FilesCb(entry)
2663 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2665 p4CmdList(["-x", "-", "print"],
2667 cb=streamP4FilesCbSelf)
2670 if self.stream_file.has_key('depotFile'):
2671 self.streamOneP4File(self.stream_file, self.stream_contents)
2673 def make_email(self, userid):
2674 if userid in self.users:
2675 return self.users[userid]
2677 return "%s <a@b>" % userid
2679 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2680 """ Stream a p4 tag.
2681 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2685 print "writing tag %s for commit %s" % (labelName, commit)
2686 gitStream.write("tag %s\n" % labelName)
2687 gitStream.write("from %s\n" % commit)
2689 if labelDetails.has_key('Owner'):
2690 owner = labelDetails["Owner"]
2694 # Try to use the owner of the p4 label, or failing that,
2695 # the current p4 user id.
2697 email = self.make_email(owner)
2699 email = self.make_email(self.p4UserId())
2700 tagger = "%s %s %s" % (email, epoch, self.tz)
2702 gitStream.write("tagger %s\n" % tagger)
2704 print "labelDetails=",labelDetails
2705 if labelDetails.has_key('Description'):
2706 description = labelDetails['Description']
2708 description = 'Label from git p4'
2710 gitStream.write("data %d\n" % len(description))
2711 gitStream.write(description)
2712 gitStream.write("\n")
2714 def inClientSpec(self, path):
2715 if not self.clientSpecDirs:
2717 inClientSpec = self.clientSpecDirs.map_in_client(path)
2718 if not inClientSpec and self.verbose:
2719 print('Ignoring file outside of client spec: {0}'.format(path))
2722 def hasBranchPrefix(self, path):
2723 if not self.branchPrefixes:
2725 hasPrefix = [p for p in self.branchPrefixes
2726 if p4PathStartsWith(path, p)]
2727 if not hasPrefix and self.verbose:
2728 print('Ignoring file outside of prefix: {0}'.format(path))
2731 def commit(self, details, files, branch, parent = ""):
2732 epoch = details["time"]
2733 author = details["user"]
2734 jobs = self.extractJobsFromCommit(details)
2737 print('commit into {0}'.format(branch))
2739 if self.clientSpecDirs:
2740 self.clientSpecDirs.update_client_spec_path_cache(files)
2742 files = [f for f in files
2743 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2745 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2746 print('Ignoring revision {0} as it would produce an empty commit.'
2747 .format(details['change']))
2750 self.gitStream.write("commit %s\n" % branch)
2751 self.gitStream.write("mark :%s\n" % details["change"])
2752 self.committedChanges.add(int(details["change"]))
2754 if author not in self.users:
2755 self.getUserMapFromPerforceServer()
2756 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2758 self.gitStream.write("committer %s\n" % committer)
2760 self.gitStream.write("data <<EOT\n")
2761 self.gitStream.write(details["desc"])
2763 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2764 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2765 (','.join(self.branchPrefixes), details["change"]))
2766 if len(details['options']) > 0:
2767 self.gitStream.write(": options = %s" % details['options'])
2768 self.gitStream.write("]\nEOT\n\n")
2772 print "parent %s" % parent
2773 self.gitStream.write("from %s\n" % parent)
2775 self.streamP4Files(files)
2776 self.gitStream.write("\n")
2778 change = int(details["change"])
2780 if self.labels.has_key(change):
2781 label = self.labels[change]
2782 labelDetails = label[0]
2783 labelRevisions = label[1]
2785 print "Change %s is labelled %s" % (change, labelDetails)
2787 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2788 for p in self.branchPrefixes])
2790 if len(files) == len(labelRevisions):
2794 if info["action"] in self.delete_actions:
2796 cleanedFiles[info["depotFile"]] = info["rev"]
2798 if cleanedFiles == labelRevisions:
2799 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2803 print ("Tag %s does not match with change %s: files do not match."
2804 % (labelDetails["label"], change))
2808 print ("Tag %s does not match with change %s: file count is different."
2809 % (labelDetails["label"], change))
2811 # Build a dictionary of changelists and labels, for "detect-labels" option.
2812 def getLabels(self):
2815 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2816 if len(l) > 0 and not self.silent:
2817 print "Finding files belonging to labels in %s" % `self.depotPaths`
2820 label = output["label"]
2824 print "Querying files for label %s" % label
2825 for file in p4CmdList(["files"] +
2826 ["%s...@%s" % (p, label)
2827 for p in self.depotPaths]):
2828 revisions[file["depotFile"]] = file["rev"]
2829 change = int(file["change"])
2830 if change > newestChange:
2831 newestChange = change
2833 self.labels[newestChange] = [output, revisions]
2836 print "Label changes: %s" % self.labels.keys()
2838 # Import p4 labels as git tags. A direct mapping does not
2839 # exist, so assume that if all the files are at the same revision
2840 # then we can use that, or it's something more complicated we should
2842 def importP4Labels(self, stream, p4Labels):
2844 print "import p4 labels: " + ' '.join(p4Labels)
2846 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2847 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2848 if len(validLabelRegexp) == 0:
2849 validLabelRegexp = defaultLabelRegexp
2850 m = re.compile(validLabelRegexp)
2852 for name in p4Labels:
2855 if not m.match(name):
2857 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2860 if name in ignoredP4Labels:
2863 labelDetails = p4CmdList(['label', "-o", name])[0]
2865 # get the most recent changelist for each file in this label
2866 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2867 for p in self.depotPaths])
2869 if change.has_key('change'):
2870 # find the corresponding git commit; take the oldest commit
2871 changelist = int(change['change'])
2872 if changelist in self.committedChanges:
2873 gitCommit = ":%d" % changelist # use a fast-import mark
2876 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2877 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2878 if len(gitCommit) == 0:
2879 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2882 gitCommit = gitCommit.strip()
2885 # Convert from p4 time format
2887 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2889 print "Could not convert label time %s" % labelDetails['Update']
2892 when = int(time.mktime(tmwhen))
2893 self.streamTag(stream, name, labelDetails, gitCommit, when)
2895 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2898 print "Label %s has no changelists - possibly deleted?" % name
2901 # We can't import this label; don't try again as it will get very
2902 # expensive repeatedly fetching all the files for labels that will
2903 # never be imported. If the label is moved in the future, the
2904 # ignore will need to be removed manually.
2905 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2907 def guessProjectName(self):
2908 for p in self.depotPaths:
2911 p = p[p.strip().rfind("/") + 1:]
2912 if not p.endswith("/"):
2916 def getBranchMapping(self):
2917 lostAndFoundBranches = set()
2919 user = gitConfig("git-p4.branchUser")
2921 command = "branches -u %s" % user
2923 command = "branches"
2925 for info in p4CmdList(command):
2926 details = p4Cmd(["branch", "-o", info["branch"]])
2928 while details.has_key("View%s" % viewIdx):
2929 paths = details["View%s" % viewIdx].split(" ")
2930 viewIdx = viewIdx + 1
2931 # require standard //depot/foo/... //depot/bar/... mapping
2932 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2935 destination = paths[1]
2937 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2938 source = source[len(self.depotPaths[0]):-4]
2939 destination = destination[len(self.depotPaths[0]):-4]
2941 if destination in self.knownBranches:
2943 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2944 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2947 self.knownBranches[destination] = source
2949 lostAndFoundBranches.discard(destination)
2951 if source not in self.knownBranches:
2952 lostAndFoundBranches.add(source)
2954 # Perforce does not strictly require branches to be defined, so we also
2955 # check git config for a branch list.
2957 # Example of branch definition in git config file:
2959 # branchList=main:branchA
2960 # branchList=main:branchB
2961 # branchList=branchA:branchC
2962 configBranches = gitConfigList("git-p4.branchList")
2963 for branch in configBranches:
2965 (source, destination) = branch.split(":")
2966 self.knownBranches[destination] = source
2968 lostAndFoundBranches.discard(destination)
2970 if source not in self.knownBranches:
2971 lostAndFoundBranches.add(source)
2974 for branch in lostAndFoundBranches:
2975 self.knownBranches[branch] = branch
2977 def getBranchMappingFromGitBranches(self):
2978 branches = p4BranchesInGit(self.importIntoRemotes)
2979 for branch in branches.keys():
2980 if branch == "master":
2983 branch = branch[len(self.projectName):]
2984 self.knownBranches[branch] = branch
2986 def updateOptionDict(self, d):
2988 if self.keepRepoPath:
2989 option_keys['keepRepoPath'] = 1
2991 d["options"] = ' '.join(sorted(option_keys.keys()))
2993 def readOptions(self, d):
2994 self.keepRepoPath = (d.has_key('options')
2995 and ('keepRepoPath' in d['options']))
2997 def gitRefForBranch(self, branch):
2998 if branch == "main":
2999 return self.refPrefix + "master"
3001 if len(branch) <= 0:
3004 return self.refPrefix + self.projectName + branch
3006 def gitCommitByP4Change(self, ref, change):
3008 print "looking in ref " + ref + " for change %s using bisect..." % change
3011 latestCommit = parseRevision(ref)
3015 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3016 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3021 log = extractLogMessageFromGitCommit(next)
3022 settings = extractSettingsGitLog(log)
3023 currentChange = int(settings['change'])
3025 print "current change %s" % currentChange
3027 if currentChange == change:
3029 print "found %s" % next
3032 if currentChange < change:
3033 earliestCommit = "^%s" % next
3035 latestCommit = "%s" % next
3039 def importNewBranch(self, branch, maxChange):
3040 # make fast-import flush all changes to disk and update the refs using the checkpoint
3041 # command so that we can try to find the branch parent in the git history
3042 self.gitStream.write("checkpoint\n\n");
3043 self.gitStream.flush();
3044 branchPrefix = self.depotPaths[0] + branch + "/"
3045 range = "@1,%s" % maxChange
3046 #print "prefix" + branchPrefix
3047 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3048 if len(changes) <= 0:
3050 firstChange = changes[0]
3051 #print "first change in branch: %s" % firstChange
3052 sourceBranch = self.knownBranches[branch]
3053 sourceDepotPath = self.depotPaths[0] + sourceBranch
3054 sourceRef = self.gitRefForBranch(sourceBranch)
3055 #print "source " + sourceBranch
3057 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3058 #print "branch parent: %s" % branchParentChange
3059 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3060 if len(gitParent) > 0:
3061 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3062 #print "parent git commit: %s" % gitParent
3064 self.importChanges(changes)
3067 def searchParent(self, parent, branch, target):
3069 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3070 "--no-merges", parent]):
3072 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3075 print "Found parent of %s in commit %s" % (branch, blob)
3082 def importChanges(self, changes):
3084 for change in changes:
3085 description = p4_describe(change)
3086 self.updateOptionDict(description)
3089 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3094 if self.detectBranches:
3095 branches = self.splitFilesIntoBranches(description)
3096 for branch in branches.keys():
3098 branchPrefix = self.depotPaths[0] + branch + "/"
3099 self.branchPrefixes = [ branchPrefix ]
3103 filesForCommit = branches[branch]
3106 print "branch is %s" % branch
3108 self.updatedBranches.add(branch)
3110 if branch not in self.createdBranches:
3111 self.createdBranches.add(branch)
3112 parent = self.knownBranches[branch]
3113 if parent == branch:
3116 fullBranch = self.projectName + branch
3117 if fullBranch not in self.p4BranchesInGit:
3119 print("\n Importing new branch %s" % fullBranch);
3120 if self.importNewBranch(branch, change - 1):
3122 self.p4BranchesInGit.append(fullBranch)
3124 print("\n Resuming with change %s" % change);
3127 print "parent determined through known branches: %s" % parent
3129 branch = self.gitRefForBranch(branch)
3130 parent = self.gitRefForBranch(parent)
3133 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3135 if len(parent) == 0 and branch in self.initialParents:
3136 parent = self.initialParents[branch]
3137 del self.initialParents[branch]
3141 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3143 print "Creating temporary branch: " + tempBranch
3144 self.commit(description, filesForCommit, tempBranch)
3145 self.tempBranches.append(tempBranch)
3147 blob = self.searchParent(parent, branch, tempBranch)
3149 self.commit(description, filesForCommit, branch, blob)
3152 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3153 self.commit(description, filesForCommit, branch, parent)
3155 files = self.extractFilesFromCommit(description)
3156 self.commit(description, files, self.branch,
3158 # only needed once, to connect to the previous commit
3159 self.initialParent = ""
3161 print self.gitError.read()
3164 def importHeadRevision(self, revision):
3165 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3168 details["user"] = "git perforce import user"
3169 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3170 % (' '.join(self.depotPaths), revision))
3171 details["change"] = revision
3175 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3177 for info in p4CmdList(["files"] + fileArgs):
3179 if 'code' in info and info['code'] == 'error':
3180 sys.stderr.write("p4 returned an error: %s\n"
3182 if info['data'].find("must refer to client") >= 0:
3183 sys.stderr.write("This particular p4 error is misleading.\n")
3184 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3185 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3187 if 'p4ExitCode' in info:
3188 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3192 change = int(info["change"])
3193 if change > newestRevision:
3194 newestRevision = change
3196 if info["action"] in self.delete_actions:
3197 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3198 #fileCnt = fileCnt + 1
3201 for prop in ["depotFile", "rev", "action", "type" ]:
3202 details["%s%s" % (prop, fileCnt)] = info[prop]
3204 fileCnt = fileCnt + 1
3206 details["change"] = newestRevision
3208 # Use time from top-most change so that all git p4 clones of
3209 # the same p4 repo have the same commit SHA1s.
3210 res = p4_describe(newestRevision)
3211 details["time"] = res["time"]
3213 self.updateOptionDict(details)
3215 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3217 print "IO error with git fast-import. Is your git version recent enough?"
3218 print self.gitError.read()
3221 def run(self, args):
3222 self.depotPaths = []
3223 self.changeRange = ""
3224 self.previousDepotPaths = []
3225 self.hasOrigin = False
3227 # map from branch depot path to parent branch
3228 self.knownBranches = {}
3229 self.initialParents = {}
3231 if self.importIntoRemotes:
3232 self.refPrefix = "refs/remotes/p4/"
3234 self.refPrefix = "refs/heads/p4/"
3236 if self.syncWithOrigin:
3237 self.hasOrigin = originP4BranchesExist()
3240 print 'Syncing with origin first, using "git fetch origin"'
3241 system("git fetch origin")
3243 branch_arg_given = bool(self.branch)
3244 if len(self.branch) == 0:
3245 self.branch = self.refPrefix + "master"
3246 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3247 system("git update-ref %s refs/heads/p4" % self.branch)
3248 system("git branch -D p4")
3250 # accept either the command-line option, or the configuration variable
3251 if self.useClientSpec:
3252 # will use this after clone to set the variable
3253 self.useClientSpec_from_options = True
3255 if gitConfigBool("git-p4.useclientspec"):
3256 self.useClientSpec = True
3257 if self.useClientSpec:
3258 self.clientSpecDirs = getClientSpec()
3260 # TODO: should always look at previous commits,
3261 # merge with previous imports, if possible.
3264 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3266 # branches holds mapping from branch name to sha1
3267 branches = p4BranchesInGit(self.importIntoRemotes)
3269 # restrict to just this one, disabling detect-branches
3270 if branch_arg_given:
3271 short = self.branch.split("/")[-1]
3272 if short in branches:
3273 self.p4BranchesInGit = [ short ]
3275 self.p4BranchesInGit = branches.keys()
3277 if len(self.p4BranchesInGit) > 1:
3279 print "Importing from/into multiple branches"
3280 self.detectBranches = True
3281 for branch in branches.keys():
3282 self.initialParents[self.refPrefix + branch] = \
3286 print "branches: %s" % self.p4BranchesInGit
3289 for branch in self.p4BranchesInGit:
3290 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3292 settings = extractSettingsGitLog(logMsg)
3294 self.readOptions(settings)
3295 if (settings.has_key('depot-paths')
3296 and settings.has_key ('change')):
3297 change = int(settings['change']) + 1
3298 p4Change = max(p4Change, change)
3300 depotPaths = sorted(settings['depot-paths'])
3301 if self.previousDepotPaths == []:
3302 self.previousDepotPaths = depotPaths
3305 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3306 prev_list = prev.split("/")
3307 cur_list = cur.split("/")
3308 for i in range(0, min(len(cur_list), len(prev_list))):
3309 if cur_list[i] <> prev_list[i]:
3313 paths.append ("/".join(cur_list[:i + 1]))
3315 self.previousDepotPaths = paths
3318 self.depotPaths = sorted(self.previousDepotPaths)
3319 self.changeRange = "@%s,#head" % p4Change
3320 if not self.silent and not self.detectBranches:
3321 print "Performing incremental import into %s git branch" % self.branch
3323 # accept multiple ref name abbreviations:
3324 # refs/foo/bar/branch -> use it exactly
3325 # p4/branch -> prepend refs/remotes/ or refs/heads/
3326 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3327 if not self.branch.startswith("refs/"):
3328 if self.importIntoRemotes:
3329 prepend = "refs/remotes/"
3331 prepend = "refs/heads/"
3332 if not self.branch.startswith("p4/"):
3334 self.branch = prepend + self.branch
3336 if len(args) == 0 and self.depotPaths:
3338 print "Depot paths: %s" % ' '.join(self.depotPaths)
3340 if self.depotPaths and self.depotPaths != args:
3341 print ("previous import used depot path %s and now %s was specified. "
3342 "This doesn't work!" % (' '.join (self.depotPaths),
3346 self.depotPaths = sorted(args)
3351 # Make sure no revision specifiers are used when --changesfile
3353 bad_changesfile = False
3354 if len(self.changesFile) > 0:
3355 for p in self.depotPaths:
3356 if p.find("@") >= 0 or p.find("#") >= 0:
3357 bad_changesfile = True
3360 die("Option --changesfile is incompatible with revision specifiers")
3363 for p in self.depotPaths:
3364 if p.find("@") != -1:
3365 atIdx = p.index("@")
3366 self.changeRange = p[atIdx:]
3367 if self.changeRange == "@all":
3368 self.changeRange = ""
3369 elif ',' not in self.changeRange:
3370 revision = self.changeRange
3371 self.changeRange = ""
3373 elif p.find("#") != -1:
3374 hashIdx = p.index("#")
3375 revision = p[hashIdx:]
3377 elif self.previousDepotPaths == []:
3378 # pay attention to changesfile, if given, else import
3379 # the entire p4 tree at the head revision
3380 if len(self.changesFile) == 0:
3383 p = re.sub ("\.\.\.$", "", p)
3384 if not p.endswith("/"):
3389 self.depotPaths = newPaths
3391 # --detect-branches may change this for each branch
3392 self.branchPrefixes = self.depotPaths
3394 self.loadUserMapFromCache()
3396 if self.detectLabels:
3399 if self.detectBranches:
3400 ## FIXME - what's a P4 projectName ?
3401 self.projectName = self.guessProjectName()
3404 self.getBranchMappingFromGitBranches()
3406 self.getBranchMapping()
3408 print "p4-git branches: %s" % self.p4BranchesInGit
3409 print "initial parents: %s" % self.initialParents
3410 for b in self.p4BranchesInGit:
3414 b = b[len(self.projectName):]
3415 self.createdBranches.add(b)
3417 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3419 self.importProcess = subprocess.Popen(["git", "fast-import"],
3420 stdin=subprocess.PIPE,
3421 stdout=subprocess.PIPE,
3422 stderr=subprocess.PIPE);
3423 self.gitOutput = self.importProcess.stdout
3424 self.gitStream = self.importProcess.stdin
3425 self.gitError = self.importProcess.stderr
3428 self.importHeadRevision(revision)
3432 if len(self.changesFile) > 0:
3433 output = open(self.changesFile).readlines()
3436 changeSet.add(int(line))
3438 for change in changeSet:
3439 changes.append(change)
3443 # catch "git p4 sync" with no new branches, in a repo that
3444 # does not have any existing p4 branches
3446 if not self.p4BranchesInGit:
3447 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3449 # The default branch is master, unless --branch is used to
3450 # specify something else. Make sure it exists, or complain
3451 # nicely about how to use --branch.
3452 if not self.detectBranches:
3453 if not branch_exists(self.branch):
3454 if branch_arg_given:
3455 die("Error: branch %s does not exist." % self.branch)
3457 die("Error: no branch %s; perhaps specify one with --branch." %
3461 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3463 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3465 if len(self.maxChanges) > 0:
3466 changes = changes[:min(int(self.maxChanges), len(changes))]
3468 if len(changes) == 0:
3470 print "No changes to import!"
3472 if not self.silent and not self.detectBranches:
3473 print "Import destination: %s" % self.branch
3475 self.updatedBranches = set()
3477 if not self.detectBranches:
3479 # start a new branch
3480 self.initialParent = ""
3482 # build on a previous revision
3483 self.initialParent = parseRevision(self.branch)
3485 self.importChanges(changes)
3489 if len(self.updatedBranches) > 0:
3490 sys.stdout.write("Updated branches: ")
3491 for b in self.updatedBranches:
3492 sys.stdout.write("%s " % b)
3493 sys.stdout.write("\n")
3495 if gitConfigBool("git-p4.importLabels"):
3496 self.importLabels = True
3498 if self.importLabels:
3499 p4Labels = getP4Labels(self.depotPaths)
3500 gitTags = getGitTags()
3502 missingP4Labels = p4Labels - gitTags
3503 self.importP4Labels(self.gitStream, missingP4Labels)
3505 self.gitStream.close()
3506 if self.importProcess.wait() != 0:
3507 die("fast-import failed: %s" % self.gitError.read())
3508 self.gitOutput.close()
3509 self.gitError.close()
3511 # Cleanup temporary branches created during import
3512 if self.tempBranches != []:
3513 for branch in self.tempBranches:
3514 read_pipe("git update-ref -d %s" % branch)
3515 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3517 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3518 # a convenient shortcut refname "p4".
3519 if self.importIntoRemotes:
3520 head_ref = self.refPrefix + "HEAD"
3521 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3522 system(["git", "symbolic-ref", head_ref, self.branch])
3526 class P4Rebase(Command):
3528 Command.__init__(self)
3530 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3532 self.importLabels = False
3533 self.description = ("Fetches the latest revision from perforce and "
3534 + "rebases the current work (branch) against it")
3536 def run(self, args):
3538 sync.importLabels = self.importLabels
3541 return self.rebase()
3544 if os.system("git update-index --refresh") != 0:
3545 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3546 if len(read_pipe("git diff-index HEAD --")) > 0:
3547 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3549 [upstream, settings] = findUpstreamBranchPoint()
3550 if len(upstream) == 0:
3551 die("Cannot find upstream branchpoint for rebase")
3553 # the branchpoint may be p4/foo~3, so strip off the parent
3554 upstream = re.sub("~[0-9]+$", "", upstream)
3556 print "Rebasing the current branch onto %s" % upstream
3557 oldHead = read_pipe("git rev-parse HEAD").strip()
3558 system("git rebase %s" % upstream)
3559 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3562 class P4Clone(P4Sync):
3564 P4Sync.__init__(self)
3565 self.description = "Creates a new git repository and imports from Perforce into it"
3566 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3568 optparse.make_option("--destination", dest="cloneDestination",
3569 action='store', default=None,
3570 help="where to leave result of the clone"),
3571 optparse.make_option("--bare", dest="cloneBare",
3572 action="store_true", default=False),
3574 self.cloneDestination = None
3575 self.needsGit = False
3576 self.cloneBare = False
3578 def defaultDestination(self, args):
3579 ## TODO: use common prefix of args?
3581 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3582 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3583 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3584 depotDir = re.sub(r"/$", "", depotDir)
3585 return os.path.split(depotDir)[1]
3587 def run(self, args):
3591 if self.keepRepoPath and not self.cloneDestination:
3592 sys.stderr.write("Must specify destination for --keep-path\n")
3597 if not self.cloneDestination and len(depotPaths) > 1:
3598 self.cloneDestination = depotPaths[-1]
3599 depotPaths = depotPaths[:-1]
3601 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3602 for p in depotPaths:
3603 if not p.startswith("//"):
3604 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3607 if not self.cloneDestination:
3608 self.cloneDestination = self.defaultDestination(args)
3610 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3612 if not os.path.exists(self.cloneDestination):
3613 os.makedirs(self.cloneDestination)
3614 chdir(self.cloneDestination)
3616 init_cmd = [ "git", "init" ]
3618 init_cmd.append("--bare")
3619 retcode = subprocess.call(init_cmd)
3621 raise CalledProcessError(retcode, init_cmd)
3623 if not P4Sync.run(self, depotPaths):
3626 # create a master branch and check out a work tree
3627 if gitBranchExists(self.branch):
3628 system([ "git", "branch", "master", self.branch ])
3629 if not self.cloneBare:
3630 system([ "git", "checkout", "-f" ])
3632 print 'Not checking out any branch, use ' \
3633 '"git checkout -q -b master <branch>"'
3635 # auto-set this variable if invoked with --use-client-spec
3636 if self.useClientSpec_from_options:
3637 system("git config --bool git-p4.useclientspec true")
3641 class P4Branches(Command):
3643 Command.__init__(self)
3645 self.description = ("Shows the git branches that hold imports and their "
3646 + "corresponding perforce depot paths")
3647 self.verbose = False
3649 def run(self, args):
3650 if originP4BranchesExist():
3651 createOrUpdateBranchesFromOrigin()
3653 cmdline = "git rev-parse --symbolic "
3654 cmdline += " --remotes"
3656 for line in read_pipe_lines(cmdline):
3659 if not line.startswith('p4/') or line == "p4/HEAD":
3663 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3664 settings = extractSettingsGitLog(log)
3666 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3669 class HelpFormatter(optparse.IndentedHelpFormatter):
3671 optparse.IndentedHelpFormatter.__init__(self)
3673 def format_description(self, description):
3675 return description + "\n"
3679 def printUsage(commands):
3680 print "usage: %s <command> [options]" % sys.argv[0]
3682 print "valid commands: %s" % ", ".join(commands)
3684 print "Try %s <command> --help for command specific help." % sys.argv[0]
3689 "submit" : P4Submit,
3690 "commit" : P4Submit,
3692 "rebase" : P4Rebase,
3694 "rollback" : P4RollBack,
3695 "branches" : P4Branches
3700 if len(sys.argv[1:]) == 0:
3701 printUsage(commands.keys())
3704 cmdName = sys.argv[1]
3706 klass = commands[cmdName]
3709 print "unknown command %s" % cmdName
3711 printUsage(commands.keys())
3714 options = cmd.options
3715 cmd.gitdir = os.environ.get("GIT_DIR", None)
3719 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3721 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3723 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3725 description = cmd.description,
3726 formatter = HelpFormatter())
3728 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3730 verbose = cmd.verbose
3732 if cmd.gitdir == None:
3733 cmd.gitdir = os.path.abspath(".git")
3734 if not isValidGitDir(cmd.gitdir):
3735 # "rev-parse --git-dir" without arguments will try $PWD/.git
3736 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3737 if os.path.exists(cmd.gitdir):
3738 cdup = read_pipe("git rev-parse --show-cdup").strip()
3742 if not isValidGitDir(cmd.gitdir):
3743 if isValidGitDir(cmd.gitdir + "/.git"):
3744 cmd.gitdir += "/.git"
3746 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3748 # so git commands invoked from the P4 workspace will succeed
3749 os.environ["GIT_DIR"] = cmd.gitdir
3751 if not cmd.run(args):
3756 if __name__ == '__main__':