.mailmap: record canonical email for Richard Hansen
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31     from subprocess import CalledProcessError
32 except ImportError:
33     # from python2.7:subprocess.py
34     # Exception classes used by this module.
35     class CalledProcessError(Exception):
36         """This exception is raised when a process run by check_call() returns
37         a non-zero exit status.  The exit status will be stored in the
38         returncode attribute."""
39         def __init__(self, returncode, cmd):
40             self.returncode = returncode
41             self.cmd = cmd
42         def __str__(self):
43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54     """Build a suitable p4 command line.
55
56     This consolidates building and returning a p4 command line into one
57     location. It means that hooking into the environment, or other configuration
58     can be done more easily.
59     """
60     real_cmd = ["p4"]
61
62     user = gitConfig("git-p4.user")
63     if len(user) > 0:
64         real_cmd += ["-u",user]
65
66     password = gitConfig("git-p4.password")
67     if len(password) > 0:
68         real_cmd += ["-P", password]
69
70     port = gitConfig("git-p4.port")
71     if len(port) > 0:
72         real_cmd += ["-p", port]
73
74     host = gitConfig("git-p4.host")
75     if len(host) > 0:
76         real_cmd += ["-H", host]
77
78     client = gitConfig("git-p4.client")
79     if len(client) > 0:
80         real_cmd += ["-c", client]
81
82     retries = gitConfigInt("git-p4.retries")
83     if retries is None:
84         # Perform 3 retries by default
85         retries = 3
86     real_cmd += ["-r", str(retries)]
87
88     if isinstance(cmd,basestring):
89         real_cmd = ' '.join(real_cmd) + ' ' + cmd
90     else:
91         real_cmd += cmd
92     return real_cmd
93
94 def git_dir(path):
95     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
96         This won't automatically add ".git" to a directory.
97     """
98     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
99     if not d or len(d) == 0:
100         return None
101     else:
102         return d
103
104 def chdir(path, is_client_path=False):
105     """Do chdir to the given path, and set the PWD environment
106        variable for use by P4.  It does not look at getcwd() output.
107        Since we're not using the shell, it is necessary to set the
108        PWD environment variable explicitly.
109
110        Normally, expand the path to force it to be absolute.  This
111        addresses the use of relative path names inside P4 settings,
112        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
113        as given; it looks for .p4config using PWD.
114
115        If is_client_path, the path was handed to us directly by p4,
116        and may be a symbolic link.  Do not call os.getcwd() in this
117        case, because it will cause p4 to think that PWD is not inside
118        the client path.
119        """
120
121     os.chdir(path)
122     if not is_client_path:
123         path = os.getcwd()
124     os.environ['PWD'] = path
125
126 def calcDiskFree():
127     """Return free space in bytes on the disk of the given dirname."""
128     if platform.system() == 'Windows':
129         free_bytes = ctypes.c_ulonglong(0)
130         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
131         return free_bytes.value
132     else:
133         st = os.statvfs(os.getcwd())
134         return st.f_bavail * st.f_frsize
135
136 def die(msg):
137     if verbose:
138         raise Exception(msg)
139     else:
140         sys.stderr.write(msg + "\n")
141         sys.exit(1)
142
143 def write_pipe(c, stdin):
144     if verbose:
145         sys.stderr.write('Writing pipe: %s\n' % str(c))
146
147     expand = isinstance(c,basestring)
148     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
149     pipe = p.stdin
150     val = pipe.write(stdin)
151     pipe.close()
152     if p.wait():
153         die('Command failed: %s' % str(c))
154
155     return val
156
157 def p4_write_pipe(c, stdin):
158     real_cmd = p4_build_cmd(c)
159     return write_pipe(real_cmd, stdin)
160
161 def read_pipe(c, ignore_error=False):
162     if verbose:
163         sys.stderr.write('Reading pipe: %s\n' % str(c))
164
165     expand = isinstance(c,basestring)
166     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
167     (out, err) = p.communicate()
168     if p.returncode != 0 and not ignore_error:
169         die('Command failed: %s\nError: %s' % (str(c), err))
170     return out
171
172 def p4_read_pipe(c, ignore_error=False):
173     real_cmd = p4_build_cmd(c)
174     return read_pipe(real_cmd, ignore_error)
175
176 def read_pipe_lines(c):
177     if verbose:
178         sys.stderr.write('Reading pipe: %s\n' % str(c))
179
180     expand = isinstance(c, basestring)
181     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
182     pipe = p.stdout
183     val = pipe.readlines()
184     if pipe.close() or p.wait():
185         die('Command failed: %s' % str(c))
186
187     return val
188
189 def p4_read_pipe_lines(c):
190     """Specifically invoke p4 on the command supplied. """
191     real_cmd = p4_build_cmd(c)
192     return read_pipe_lines(real_cmd)
193
194 def p4_has_command(cmd):
195     """Ask p4 for help on this command.  If it returns an error, the
196        command does not exist in this version of p4."""
197     real_cmd = p4_build_cmd(["help", cmd])
198     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
199                                    stderr=subprocess.PIPE)
200     p.communicate()
201     return p.returncode == 0
202
203 def p4_has_move_command():
204     """See if the move command exists, that it supports -k, and that
205        it has not been administratively disabled.  The arguments
206        must be correct, but the filenames do not have to exist.  Use
207        ones with wildcards so even if they exist, it will fail."""
208
209     if not p4_has_command("move"):
210         return False
211     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
212     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
213     (out, err) = p.communicate()
214     # return code will be 1 in either case
215     if err.find("Invalid option") >= 0:
216         return False
217     if err.find("disabled") >= 0:
218         return False
219     # assume it failed because @... was invalid changelist
220     return True
221
222 def system(cmd, ignore_error=False):
223     expand = isinstance(cmd,basestring)
224     if verbose:
225         sys.stderr.write("executing %s\n" % str(cmd))
226     retcode = subprocess.call(cmd, shell=expand)
227     if retcode and not ignore_error:
228         raise CalledProcessError(retcode, cmd)
229
230     return retcode
231
232 def p4_system(cmd):
233     """Specifically invoke p4 as the system command. """
234     real_cmd = p4_build_cmd(cmd)
235     expand = isinstance(real_cmd, basestring)
236     retcode = subprocess.call(real_cmd, shell=expand)
237     if retcode:
238         raise CalledProcessError(retcode, real_cmd)
239
240 _p4_version_string = None
241 def p4_version_string():
242     """Read the version string, showing just the last line, which
243        hopefully is the interesting version bit.
244
245        $ p4 -V
246        Perforce - The Fast Software Configuration Management System.
247        Copyright 1995-2011 Perforce Software.  All rights reserved.
248        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
249     """
250     global _p4_version_string
251     if not _p4_version_string:
252         a = p4_read_pipe_lines(["-V"])
253         _p4_version_string = a[-1].rstrip()
254     return _p4_version_string
255
256 def p4_integrate(src, dest):
257     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
258
259 def p4_sync(f, *options):
260     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
261
262 def p4_add(f):
263     # forcibly add file names with wildcards
264     if wildcard_present(f):
265         p4_system(["add", "-f", f])
266     else:
267         p4_system(["add", f])
268
269 def p4_delete(f):
270     p4_system(["delete", wildcard_encode(f)])
271
272 def p4_edit(f, *options):
273     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
274
275 def p4_revert(f):
276     p4_system(["revert", wildcard_encode(f)])
277
278 def p4_reopen(type, f):
279     p4_system(["reopen", "-t", type, wildcard_encode(f)])
280
281 def p4_reopen_in_change(changelist, files):
282     cmd = ["reopen", "-c", str(changelist)] + files
283     p4_system(cmd)
284
285 def p4_move(src, dest):
286     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
287
288 def p4_last_change():
289     results = p4CmdList(["changes", "-m", "1"])
290     return int(results[0]['change'])
291
292 def p4_describe(change):
293     """Make sure it returns a valid result by checking for
294        the presence of field "time".  Return a dict of the
295        results."""
296
297     ds = p4CmdList(["describe", "-s", str(change)])
298     if len(ds) != 1:
299         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
300
301     d = ds[0]
302
303     if "p4ExitCode" in d:
304         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
305                                                       str(d)))
306     if "code" in d:
307         if d["code"] == "error":
308             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
309
310     if "time" not in d:
311         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
312
313     return d
314
315 #
316 # Canonicalize the p4 type and return a tuple of the
317 # base type, plus any modifiers.  See "p4 help filetypes"
318 # for a list and explanation.
319 #
320 def split_p4_type(p4type):
321
322     p4_filetypes_historical = {
323         "ctempobj": "binary+Sw",
324         "ctext": "text+C",
325         "cxtext": "text+Cx",
326         "ktext": "text+k",
327         "kxtext": "text+kx",
328         "ltext": "text+F",
329         "tempobj": "binary+FSw",
330         "ubinary": "binary+F",
331         "uresource": "resource+F",
332         "uxbinary": "binary+Fx",
333         "xbinary": "binary+x",
334         "xltext": "text+Fx",
335         "xtempobj": "binary+Swx",
336         "xtext": "text+x",
337         "xunicode": "unicode+x",
338         "xutf16": "utf16+x",
339     }
340     if p4type in p4_filetypes_historical:
341         p4type = p4_filetypes_historical[p4type]
342     mods = ""
343     s = p4type.split("+")
344     base = s[0]
345     mods = ""
346     if len(s) > 1:
347         mods = s[1]
348     return (base, mods)
349
350 #
351 # return the raw p4 type of a file (text, text+ko, etc)
352 #
353 def p4_type(f):
354     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
355     return results[0]['headType']
356
357 #
358 # Given a type base and modifier, return a regexp matching
359 # the keywords that can be expanded in the file
360 #
361 def p4_keywords_regexp_for_type(base, type_mods):
362     if base in ("text", "unicode", "binary"):
363         kwords = None
364         if "ko" in type_mods:
365             kwords = 'Id|Header'
366         elif "k" in type_mods:
367             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
368         else:
369             return None
370         pattern = r"""
371             \$              # Starts with a dollar, followed by...
372             (%s)            # one of the keywords, followed by...
373             (:[^$\n]+)?     # possibly an old expansion, followed by...
374             \$              # another dollar
375             """ % kwords
376         return pattern
377     else:
378         return None
379
380 #
381 # Given a file, return a regexp matching the possible
382 # RCS keywords that will be expanded, or None for files
383 # with kw expansion turned off.
384 #
385 def p4_keywords_regexp_for_file(file):
386     if not os.path.exists(file):
387         return None
388     else:
389         (type_base, type_mods) = split_p4_type(p4_type(file))
390         return p4_keywords_regexp_for_type(type_base, type_mods)
391
392 def setP4ExecBit(file, mode):
393     # Reopens an already open file and changes the execute bit to match
394     # the execute bit setting in the passed in mode.
395
396     p4Type = "+x"
397
398     if not isModeExec(mode):
399         p4Type = getP4OpenedType(file)
400         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
401         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
402         if p4Type[-1] == "+":
403             p4Type = p4Type[0:-1]
404
405     p4_reopen(p4Type, file)
406
407 def getP4OpenedType(file):
408     # Returns the perforce file type for the given file.
409
410     result = p4_read_pipe(["opened", wildcard_encode(file)])
411     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
412     if match:
413         return match.group(1)
414     else:
415         die("Could not determine file type for %s (result: '%s')" % (file, result))
416
417 # Return the set of all p4 labels
418 def getP4Labels(depotPaths):
419     labels = set()
420     if isinstance(depotPaths,basestring):
421         depotPaths = [depotPaths]
422
423     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
424         label = l['label']
425         labels.add(label)
426
427     return labels
428
429 # Return the set of all git tags
430 def getGitTags():
431     gitTags = set()
432     for line in read_pipe_lines(["git", "tag"]):
433         tag = line.strip()
434         gitTags.add(tag)
435     return gitTags
436
437 def diffTreePattern():
438     # This is a simple generator for the diff tree regex pattern. This could be
439     # a class variable if this and parseDiffTreeEntry were a part of a class.
440     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
441     while True:
442         yield pattern
443
444 def parseDiffTreeEntry(entry):
445     """Parses a single diff tree entry into its component elements.
446
447     See git-diff-tree(1) manpage for details about the format of the diff
448     output. This method returns a dictionary with the following elements:
449
450     src_mode - The mode of the source file
451     dst_mode - The mode of the destination file
452     src_sha1 - The sha1 for the source file
453     dst_sha1 - The sha1 fr the destination file
454     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
455     status_score - The score for the status (applicable for 'C' and 'R'
456                    statuses). This is None if there is no score.
457     src - The path for the source file.
458     dst - The path for the destination file. This is only present for
459           copy or renames. If it is not present, this is None.
460
461     If the pattern is not matched, None is returned."""
462
463     match = diffTreePattern().next().match(entry)
464     if match:
465         return {
466             'src_mode': match.group(1),
467             'dst_mode': match.group(2),
468             'src_sha1': match.group(3),
469             'dst_sha1': match.group(4),
470             'status': match.group(5),
471             'status_score': match.group(6),
472             'src': match.group(7),
473             'dst': match.group(10)
474         }
475     return None
476
477 def isModeExec(mode):
478     # Returns True if the given git mode represents an executable file,
479     # otherwise False.
480     return mode[-3:] == "755"
481
482 def isModeExecChanged(src_mode, dst_mode):
483     return isModeExec(src_mode) != isModeExec(dst_mode)
484
485 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
486
487     if isinstance(cmd,basestring):
488         cmd = "-G " + cmd
489         expand = True
490     else:
491         cmd = ["-G"] + cmd
492         expand = False
493
494     cmd = p4_build_cmd(cmd)
495     if verbose:
496         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
497
498     # Use a temporary file to avoid deadlocks without
499     # subprocess.communicate(), which would put another copy
500     # of stdout into memory.
501     stdin_file = None
502     if stdin is not None:
503         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
504         if isinstance(stdin,basestring):
505             stdin_file.write(stdin)
506         else:
507             for i in stdin:
508                 stdin_file.write(i + '\n')
509         stdin_file.flush()
510         stdin_file.seek(0)
511
512     p4 = subprocess.Popen(cmd,
513                           shell=expand,
514                           stdin=stdin_file,
515                           stdout=subprocess.PIPE)
516
517     result = []
518     try:
519         while True:
520             entry = marshal.load(p4.stdout)
521             if cb is not None:
522                 cb(entry)
523             else:
524                 result.append(entry)
525     except EOFError:
526         pass
527     exitCode = p4.wait()
528     if exitCode != 0:
529         entry = {}
530         entry["p4ExitCode"] = exitCode
531         result.append(entry)
532
533     return result
534
535 def p4Cmd(cmd):
536     list = p4CmdList(cmd)
537     result = {}
538     for entry in list:
539         result.update(entry)
540     return result;
541
542 def p4Where(depotPath):
543     if not depotPath.endswith("/"):
544         depotPath += "/"
545     depotPathLong = depotPath + "..."
546     outputList = p4CmdList(["where", depotPathLong])
547     output = None
548     for entry in outputList:
549         if "depotFile" in entry:
550             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
551             # The base path always ends with "/...".
552             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
553                 output = entry
554                 break
555         elif "data" in entry:
556             data = entry.get("data")
557             space = data.find(" ")
558             if data[:space] == depotPath:
559                 output = entry
560                 break
561     if output == None:
562         return ""
563     if output["code"] == "error":
564         return ""
565     clientPath = ""
566     if "path" in output:
567         clientPath = output.get("path")
568     elif "data" in output:
569         data = output.get("data")
570         lastSpace = data.rfind(" ")
571         clientPath = data[lastSpace + 1:]
572
573     if clientPath.endswith("..."):
574         clientPath = clientPath[:-3]
575     return clientPath
576
577 def currentGitBranch():
578     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
579     if retcode != 0:
580         # on a detached head
581         return None
582     else:
583         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
584
585 def isValidGitDir(path):
586     return git_dir(path) != None
587
588 def parseRevision(ref):
589     return read_pipe("git rev-parse %s" % ref).strip()
590
591 def branchExists(ref):
592     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
593                      ignore_error=True)
594     return len(rev) > 0
595
596 def extractLogMessageFromGitCommit(commit):
597     logMessage = ""
598
599     ## fixme: title is first line of commit, not 1st paragraph.
600     foundTitle = False
601     for log in read_pipe_lines("git cat-file commit %s" % commit):
602        if not foundTitle:
603            if len(log) == 1:
604                foundTitle = True
605            continue
606
607        logMessage += log
608     return logMessage
609
610 def extractSettingsGitLog(log):
611     values = {}
612     for line in log.split("\n"):
613         line = line.strip()
614         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
615         if not m:
616             continue
617
618         assignments = m.group(1).split (':')
619         for a in assignments:
620             vals = a.split ('=')
621             key = vals[0].strip()
622             val = ('='.join (vals[1:])).strip()
623             if val.endswith ('\"') and val.startswith('"'):
624                 val = val[1:-1]
625
626             values[key] = val
627
628     paths = values.get("depot-paths")
629     if not paths:
630         paths = values.get("depot-path")
631     if paths:
632         values['depot-paths'] = paths.split(',')
633     return values
634
635 def gitBranchExists(branch):
636     proc = subprocess.Popen(["git", "rev-parse", branch],
637                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
638     return proc.wait() == 0;
639
640 _gitConfig = {}
641
642 def gitConfig(key, typeSpecifier=None):
643     if not _gitConfig.has_key(key):
644         cmd = [ "git", "config" ]
645         if typeSpecifier:
646             cmd += [ typeSpecifier ]
647         cmd += [ key ]
648         s = read_pipe(cmd, ignore_error=True)
649         _gitConfig[key] = s.strip()
650     return _gitConfig[key]
651
652 def gitConfigBool(key):
653     """Return a bool, using git config --bool.  It is True only if the
654        variable is set to true, and False if set to false or not present
655        in the config."""
656
657     if not _gitConfig.has_key(key):
658         _gitConfig[key] = gitConfig(key, '--bool') == "true"
659     return _gitConfig[key]
660
661 def gitConfigInt(key):
662     if not _gitConfig.has_key(key):
663         cmd = [ "git", "config", "--int", key ]
664         s = read_pipe(cmd, ignore_error=True)
665         v = s.strip()
666         try:
667             _gitConfig[key] = int(gitConfig(key, '--int'))
668         except ValueError:
669             _gitConfig[key] = None
670     return _gitConfig[key]
671
672 def gitConfigList(key):
673     if not _gitConfig.has_key(key):
674         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
675         _gitConfig[key] = s.strip().split(os.linesep)
676         if _gitConfig[key] == ['']:
677             _gitConfig[key] = []
678     return _gitConfig[key]
679
680 def p4BranchesInGit(branchesAreInRemotes=True):
681     """Find all the branches whose names start with "p4/", looking
682        in remotes or heads as specified by the argument.  Return
683        a dictionary of { branch: revision } for each one found.
684        The branch names are the short names, without any
685        "p4/" prefix."""
686
687     branches = {}
688
689     cmdline = "git rev-parse --symbolic "
690     if branchesAreInRemotes:
691         cmdline += "--remotes"
692     else:
693         cmdline += "--branches"
694
695     for line in read_pipe_lines(cmdline):
696         line = line.strip()
697
698         # only import to p4/
699         if not line.startswith('p4/'):
700             continue
701         # special symbolic ref to p4/master
702         if line == "p4/HEAD":
703             continue
704
705         # strip off p4/ prefix
706         branch = line[len("p4/"):]
707
708         branches[branch] = parseRevision(line)
709
710     return branches
711
712 def branch_exists(branch):
713     """Make sure that the given ref name really exists."""
714
715     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
716     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
717     out, _ = p.communicate()
718     if p.returncode:
719         return False
720     # expect exactly one line of output: the branch name
721     return out.rstrip() == branch
722
723 def findUpstreamBranchPoint(head = "HEAD"):
724     branches = p4BranchesInGit()
725     # map from depot-path to branch name
726     branchByDepotPath = {}
727     for branch in branches.keys():
728         tip = branches[branch]
729         log = extractLogMessageFromGitCommit(tip)
730         settings = extractSettingsGitLog(log)
731         if settings.has_key("depot-paths"):
732             paths = ",".join(settings["depot-paths"])
733             branchByDepotPath[paths] = "remotes/p4/" + branch
734
735     settings = None
736     parent = 0
737     while parent < 65535:
738         commit = head + "~%s" % parent
739         log = extractLogMessageFromGitCommit(commit)
740         settings = extractSettingsGitLog(log)
741         if settings.has_key("depot-paths"):
742             paths = ",".join(settings["depot-paths"])
743             if branchByDepotPath.has_key(paths):
744                 return [branchByDepotPath[paths], settings]
745
746         parent = parent + 1
747
748     return ["", settings]
749
750 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
751     if not silent:
752         print ("Creating/updating branch(es) in %s based on origin branch(es)"
753                % localRefPrefix)
754
755     originPrefix = "origin/p4/"
756
757     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
758         line = line.strip()
759         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
760             continue
761
762         headName = line[len(originPrefix):]
763         remoteHead = localRefPrefix + headName
764         originHead = line
765
766         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
767         if (not original.has_key('depot-paths')
768             or not original.has_key('change')):
769             continue
770
771         update = False
772         if not gitBranchExists(remoteHead):
773             if verbose:
774                 print "creating %s" % remoteHead
775             update = True
776         else:
777             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
778             if settings.has_key('change') > 0:
779                 if settings['depot-paths'] == original['depot-paths']:
780                     originP4Change = int(original['change'])
781                     p4Change = int(settings['change'])
782                     if originP4Change > p4Change:
783                         print ("%s (%s) is newer than %s (%s). "
784                                "Updating p4 branch from origin."
785                                % (originHead, originP4Change,
786                                   remoteHead, p4Change))
787                         update = True
788                 else:
789                     print ("Ignoring: %s was imported from %s while "
790                            "%s was imported from %s"
791                            % (originHead, ','.join(original['depot-paths']),
792                               remoteHead, ','.join(settings['depot-paths'])))
793
794         if update:
795             system("git update-ref %s %s" % (remoteHead, originHead))
796
797 def originP4BranchesExist():
798         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
799
800
801 def p4ParseNumericChangeRange(parts):
802     changeStart = int(parts[0][1:])
803     if parts[1] == '#head':
804         changeEnd = p4_last_change()
805     else:
806         changeEnd = int(parts[1])
807
808     return (changeStart, changeEnd)
809
810 def chooseBlockSize(blockSize):
811     if blockSize:
812         return blockSize
813     else:
814         return defaultBlockSize
815
816 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
817     assert depotPaths
818
819     # Parse the change range into start and end. Try to find integer
820     # revision ranges as these can be broken up into blocks to avoid
821     # hitting server-side limits (maxrows, maxscanresults). But if
822     # that doesn't work, fall back to using the raw revision specifier
823     # strings, without using block mode.
824
825     if changeRange is None or changeRange == '':
826         changeStart = 1
827         changeEnd = p4_last_change()
828         block_size = chooseBlockSize(requestedBlockSize)
829     else:
830         parts = changeRange.split(',')
831         assert len(parts) == 2
832         try:
833             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
834             block_size = chooseBlockSize(requestedBlockSize)
835         except:
836             changeStart = parts[0][1:]
837             changeEnd = parts[1]
838             if requestedBlockSize:
839                 die("cannot use --changes-block-size with non-numeric revisions")
840             block_size = None
841
842     changes = set()
843
844     # Retrieve changes a block at a time, to prevent running
845     # into a MaxResults/MaxScanRows error from the server.
846
847     while True:
848         cmd = ['changes']
849
850         if block_size:
851             end = min(changeEnd, changeStart + block_size)
852             revisionRange = "%d,%d" % (changeStart, end)
853         else:
854             revisionRange = "%s,%s" % (changeStart, changeEnd)
855
856         for p in depotPaths:
857             cmd += ["%s...@%s" % (p, revisionRange)]
858
859         # Insert changes in chronological order
860         for line in reversed(p4_read_pipe_lines(cmd)):
861             changes.add(int(line.split(" ")[1]))
862
863         if not block_size:
864             break
865
866         if end >= changeEnd:
867             break
868
869         changeStart = end + 1
870
871     changes = sorted(changes)
872     return changes
873
874 def p4PathStartsWith(path, prefix):
875     # This method tries to remedy a potential mixed-case issue:
876     #
877     # If UserA adds  //depot/DirA/file1
878     # and UserB adds //depot/dira/file2
879     #
880     # we may or may not have a problem. If you have core.ignorecase=true,
881     # we treat DirA and dira as the same directory
882     if gitConfigBool("core.ignorecase"):
883         return path.lower().startswith(prefix.lower())
884     return path.startswith(prefix)
885
886 def getClientSpec():
887     """Look at the p4 client spec, create a View() object that contains
888        all the mappings, and return it."""
889
890     specList = p4CmdList("client -o")
891     if len(specList) != 1:
892         die('Output from "client -o" is %d lines, expecting 1' %
893             len(specList))
894
895     # dictionary of all client parameters
896     entry = specList[0]
897
898     # the //client/ name
899     client_name = entry["Client"]
900
901     # just the keys that start with "View"
902     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
903
904     # hold this new View
905     view = View(client_name)
906
907     # append the lines, in order, to the view
908     for view_num in range(len(view_keys)):
909         k = "View%d" % view_num
910         if k not in view_keys:
911             die("Expected view key %s missing" % k)
912         view.append(entry[k])
913
914     return view
915
916 def getClientRoot():
917     """Grab the client directory."""
918
919     output = p4CmdList("client -o")
920     if len(output) != 1:
921         die('Output from "client -o" is %d lines, expecting 1' % len(output))
922
923     entry = output[0]
924     if "Root" not in entry:
925         die('Client has no "Root"')
926
927     return entry["Root"]
928
929 #
930 # P4 wildcards are not allowed in filenames.  P4 complains
931 # if you simply add them, but you can force it with "-f", in
932 # which case it translates them into %xx encoding internally.
933 #
934 def wildcard_decode(path):
935     # Search for and fix just these four characters.  Do % last so
936     # that fixing it does not inadvertently create new %-escapes.
937     # Cannot have * in a filename in windows; untested as to
938     # what p4 would do in such a case.
939     if not platform.system() == "Windows":
940         path = path.replace("%2A", "*")
941     path = path.replace("%23", "#") \
942                .replace("%40", "@") \
943                .replace("%25", "%")
944     return path
945
946 def wildcard_encode(path):
947     # do % first to avoid double-encoding the %s introduced here
948     path = path.replace("%", "%25") \
949                .replace("*", "%2A") \
950                .replace("#", "%23") \
951                .replace("@", "%40")
952     return path
953
954 def wildcard_present(path):
955     m = re.search("[*#@%]", path)
956     return m is not None
957
958 class LargeFileSystem(object):
959     """Base class for large file system support."""
960
961     def __init__(self, writeToGitStream):
962         self.largeFiles = set()
963         self.writeToGitStream = writeToGitStream
964
965     def generatePointer(self, cloneDestination, contentFile):
966         """Return the content of a pointer file that is stored in Git instead of
967            the actual content."""
968         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
969
970     def pushFile(self, localLargeFile):
971         """Push the actual content which is not stored in the Git repository to
972            a server."""
973         assert False, "Method 'pushFile' required in " + self.__class__.__name__
974
975     def hasLargeFileExtension(self, relPath):
976         return reduce(
977             lambda a, b: a or b,
978             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
979             False
980         )
981
982     def generateTempFile(self, contents):
983         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
984         for d in contents:
985             contentFile.write(d)
986         contentFile.close()
987         return contentFile.name
988
989     def exceedsLargeFileThreshold(self, relPath, contents):
990         if gitConfigInt('git-p4.largeFileThreshold'):
991             contentsSize = sum(len(d) for d in contents)
992             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
993                 return True
994         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
995             contentsSize = sum(len(d) for d in contents)
996             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
997                 return False
998             contentTempFile = self.generateTempFile(contents)
999             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1000             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1001             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1002             zf.close()
1003             compressedContentsSize = zf.infolist()[0].compress_size
1004             os.remove(contentTempFile)
1005             os.remove(compressedContentFile.name)
1006             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1007                 return True
1008         return False
1009
1010     def addLargeFile(self, relPath):
1011         self.largeFiles.add(relPath)
1012
1013     def removeLargeFile(self, relPath):
1014         self.largeFiles.remove(relPath)
1015
1016     def isLargeFile(self, relPath):
1017         return relPath in self.largeFiles
1018
1019     def processContent(self, git_mode, relPath, contents):
1020         """Processes the content of git fast import. This method decides if a
1021            file is stored in the large file system and handles all necessary
1022            steps."""
1023         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1024             contentTempFile = self.generateTempFile(contents)
1025             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1026             if pointer_git_mode:
1027                 git_mode = pointer_git_mode
1028             if localLargeFile:
1029                 # Move temp file to final location in large file system
1030                 largeFileDir = os.path.dirname(localLargeFile)
1031                 if not os.path.isdir(largeFileDir):
1032                     os.makedirs(largeFileDir)
1033                 shutil.move(contentTempFile, localLargeFile)
1034                 self.addLargeFile(relPath)
1035                 if gitConfigBool('git-p4.largeFilePush'):
1036                     self.pushFile(localLargeFile)
1037                 if verbose:
1038                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1039         return (git_mode, contents)
1040
1041 class MockLFS(LargeFileSystem):
1042     """Mock large file system for testing."""
1043
1044     def generatePointer(self, contentFile):
1045         """The pointer content is the original content prefixed with "pointer-".
1046            The local filename of the large file storage is derived from the file content.
1047            """
1048         with open(contentFile, 'r') as f:
1049             content = next(f)
1050             gitMode = '100644'
1051             pointerContents = 'pointer-' + content
1052             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1053             return (gitMode, pointerContents, localLargeFile)
1054
1055     def pushFile(self, localLargeFile):
1056         """The remote filename of the large file storage is the same as the local
1057            one but in a different directory.
1058            """
1059         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1060         if not os.path.exists(remotePath):
1061             os.makedirs(remotePath)
1062         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1063
1064 class GitLFS(LargeFileSystem):
1065     """Git LFS as backend for the git-p4 large file system.
1066        See https://git-lfs.github.com/ for details."""
1067
1068     def __init__(self, *args):
1069         LargeFileSystem.__init__(self, *args)
1070         self.baseGitAttributes = []
1071
1072     def generatePointer(self, contentFile):
1073         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1074            mode and content which is stored in the Git repository instead of
1075            the actual content. Return also the new location of the actual
1076            content.
1077            """
1078         if os.path.getsize(contentFile) == 0:
1079             return (None, '', None)
1080
1081         pointerProcess = subprocess.Popen(
1082             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1083             stdout=subprocess.PIPE
1084         )
1085         pointerFile = pointerProcess.stdout.read()
1086         if pointerProcess.wait():
1087             os.remove(contentFile)
1088             die('git-lfs pointer command failed. Did you install the extension?')
1089
1090         # Git LFS removed the preamble in the output of the 'pointer' command
1091         # starting from version 1.2.0. Check for the preamble here to support
1092         # earlier versions.
1093         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1094         if pointerFile.startswith('Git LFS pointer for'):
1095             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1096
1097         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1098         localLargeFile = os.path.join(
1099             os.getcwd(),
1100             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1101             oid,
1102         )
1103         # LFS Spec states that pointer files should not have the executable bit set.
1104         gitMode = '100644'
1105         return (gitMode, pointerFile, localLargeFile)
1106
1107     def pushFile(self, localLargeFile):
1108         uploadProcess = subprocess.Popen(
1109             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1110         )
1111         if uploadProcess.wait():
1112             die('git-lfs push command failed. Did you define a remote?')
1113
1114     def generateGitAttributes(self):
1115         return (
1116             self.baseGitAttributes +
1117             [
1118                 '\n',
1119                 '#\n',
1120                 '# Git LFS (see https://git-lfs.github.com/)\n',
1121                 '#\n',
1122             ] +
1123             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1124                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1125             ] +
1126             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1127                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1128             ]
1129         )
1130
1131     def addLargeFile(self, relPath):
1132         LargeFileSystem.addLargeFile(self, relPath)
1133         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1134
1135     def removeLargeFile(self, relPath):
1136         LargeFileSystem.removeLargeFile(self, relPath)
1137         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1138
1139     def processContent(self, git_mode, relPath, contents):
1140         if relPath == '.gitattributes':
1141             self.baseGitAttributes = contents
1142             return (git_mode, self.generateGitAttributes())
1143         else:
1144             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1145
1146 class Command:
1147     def __init__(self):
1148         self.usage = "usage: %prog [options]"
1149         self.needsGit = True
1150         self.verbose = False
1151
1152 class P4UserMap:
1153     def __init__(self):
1154         self.userMapFromPerforceServer = False
1155         self.myP4UserId = None
1156
1157     def p4UserId(self):
1158         if self.myP4UserId:
1159             return self.myP4UserId
1160
1161         results = p4CmdList("user -o")
1162         for r in results:
1163             if r.has_key('User'):
1164                 self.myP4UserId = r['User']
1165                 return r['User']
1166         die("Could not find your p4 user id")
1167
1168     def p4UserIsMe(self, p4User):
1169         # return True if the given p4 user is actually me
1170         me = self.p4UserId()
1171         if not p4User or p4User != me:
1172             return False
1173         else:
1174             return True
1175
1176     def getUserCacheFilename(self):
1177         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1178         return home + "/.gitp4-usercache.txt"
1179
1180     def getUserMapFromPerforceServer(self):
1181         if self.userMapFromPerforceServer:
1182             return
1183         self.users = {}
1184         self.emails = {}
1185
1186         for output in p4CmdList("users"):
1187             if not output.has_key("User"):
1188                 continue
1189             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1190             self.emails[output["Email"]] = output["User"]
1191
1192         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1193         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1194             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1195             if mapUser and len(mapUser[0]) == 3:
1196                 user = mapUser[0][0]
1197                 fullname = mapUser[0][1]
1198                 email = mapUser[0][2]
1199                 self.users[user] = fullname + " <" + email + ">"
1200                 self.emails[email] = user
1201
1202         s = ''
1203         for (key, val) in self.users.items():
1204             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1205
1206         open(self.getUserCacheFilename(), "wb").write(s)
1207         self.userMapFromPerforceServer = True
1208
1209     def loadUserMapFromCache(self):
1210         self.users = {}
1211         self.userMapFromPerforceServer = False
1212         try:
1213             cache = open(self.getUserCacheFilename(), "rb")
1214             lines = cache.readlines()
1215             cache.close()
1216             for line in lines:
1217                 entry = line.strip().split("\t")
1218                 self.users[entry[0]] = entry[1]
1219         except IOError:
1220             self.getUserMapFromPerforceServer()
1221
1222 class P4Debug(Command):
1223     def __init__(self):
1224         Command.__init__(self)
1225         self.options = []
1226         self.description = "A tool to debug the output of p4 -G."
1227         self.needsGit = False
1228
1229     def run(self, args):
1230         j = 0
1231         for output in p4CmdList(args):
1232             print 'Element: %d' % j
1233             j += 1
1234             print output
1235         return True
1236
1237 class P4RollBack(Command):
1238     def __init__(self):
1239         Command.__init__(self)
1240         self.options = [
1241             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1242         ]
1243         self.description = "A tool to debug the multi-branch import. Don't use :)"
1244         self.rollbackLocalBranches = False
1245
1246     def run(self, args):
1247         if len(args) != 1:
1248             return False
1249         maxChange = int(args[0])
1250
1251         if "p4ExitCode" in p4Cmd("changes -m 1"):
1252             die("Problems executing p4");
1253
1254         if self.rollbackLocalBranches:
1255             refPrefix = "refs/heads/"
1256             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1257         else:
1258             refPrefix = "refs/remotes/"
1259             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1260
1261         for line in lines:
1262             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1263                 line = line.strip()
1264                 ref = refPrefix + line
1265                 log = extractLogMessageFromGitCommit(ref)
1266                 settings = extractSettingsGitLog(log)
1267
1268                 depotPaths = settings['depot-paths']
1269                 change = settings['change']
1270
1271                 changed = False
1272
1273                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1274                                                            for p in depotPaths]))) == 0:
1275                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1276                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1277                     continue
1278
1279                 while change and int(change) > maxChange:
1280                     changed = True
1281                     if self.verbose:
1282                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1283                     system("git update-ref %s \"%s^\"" % (ref, ref))
1284                     log = extractLogMessageFromGitCommit(ref)
1285                     settings =  extractSettingsGitLog(log)
1286
1287
1288                     depotPaths = settings['depot-paths']
1289                     change = settings['change']
1290
1291                 if changed:
1292                     print "%s rewound to %s" % (ref, change)
1293
1294         return True
1295
1296 class P4Submit(Command, P4UserMap):
1297
1298     conflict_behavior_choices = ("ask", "skip", "quit")
1299
1300     def __init__(self):
1301         Command.__init__(self)
1302         P4UserMap.__init__(self)
1303         self.options = [
1304                 optparse.make_option("--origin", dest="origin"),
1305                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1306                 # preserve the user, requires relevant p4 permissions
1307                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1308                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1309                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1310                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1311                 optparse.make_option("--conflict", dest="conflict_behavior",
1312                                      choices=self.conflict_behavior_choices),
1313                 optparse.make_option("--branch", dest="branch"),
1314                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1315                                      help="Shelve instead of submit. Shelved files are reverted, "
1316                                      "restoring the workspace to the state before the shelve"),
1317                 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1318                                      metavar="CHANGELIST",
1319                                      help="update an existing shelved changelist, implies --shelve")
1320         ]
1321         self.description = "Submit changes from git to the perforce depot."
1322         self.usage += " [name of git branch to submit into perforce depot]"
1323         self.origin = ""
1324         self.detectRenames = False
1325         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1326         self.dry_run = False
1327         self.shelve = False
1328         self.update_shelve = None
1329         self.prepare_p4_only = False
1330         self.conflict_behavior = None
1331         self.isWindows = (platform.system() == "Windows")
1332         self.exportLabels = False
1333         self.p4HasMoveCommand = p4_has_move_command()
1334         self.branch = None
1335
1336         if gitConfig('git-p4.largeFileSystem'):
1337             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1338
1339     def check(self):
1340         if len(p4CmdList("opened ...")) > 0:
1341             die("You have files opened with perforce! Close them before starting the sync.")
1342
1343     def separate_jobs_from_description(self, message):
1344         """Extract and return a possible Jobs field in the commit
1345            message.  It goes into a separate section in the p4 change
1346            specification.
1347
1348            A jobs line starts with "Jobs:" and looks like a new field
1349            in a form.  Values are white-space separated on the same
1350            line or on following lines that start with a tab.
1351
1352            This does not parse and extract the full git commit message
1353            like a p4 form.  It just sees the Jobs: line as a marker
1354            to pass everything from then on directly into the p4 form,
1355            but outside the description section.
1356
1357            Return a tuple (stripped log message, jobs string)."""
1358
1359         m = re.search(r'^Jobs:', message, re.MULTILINE)
1360         if m is None:
1361             return (message, None)
1362
1363         jobtext = message[m.start():]
1364         stripped_message = message[:m.start()].rstrip()
1365         return (stripped_message, jobtext)
1366
1367     def prepareLogMessage(self, template, message, jobs):
1368         """Edits the template returned from "p4 change -o" to insert
1369            the message in the Description field, and the jobs text in
1370            the Jobs field."""
1371         result = ""
1372
1373         inDescriptionSection = False
1374
1375         for line in template.split("\n"):
1376             if line.startswith("#"):
1377                 result += line + "\n"
1378                 continue
1379
1380             if inDescriptionSection:
1381                 if line.startswith("Files:") or line.startswith("Jobs:"):
1382                     inDescriptionSection = False
1383                     # insert Jobs section
1384                     if jobs:
1385                         result += jobs + "\n"
1386                 else:
1387                     continue
1388             else:
1389                 if line.startswith("Description:"):
1390                     inDescriptionSection = True
1391                     line += "\n"
1392                     for messageLine in message.split("\n"):
1393                         line += "\t" + messageLine + "\n"
1394
1395             result += line + "\n"
1396
1397         return result
1398
1399     def patchRCSKeywords(self, file, pattern):
1400         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1401         (handle, outFileName) = tempfile.mkstemp(dir='.')
1402         try:
1403             outFile = os.fdopen(handle, "w+")
1404             inFile = open(file, "r")
1405             regexp = re.compile(pattern, re.VERBOSE)
1406             for line in inFile.readlines():
1407                 line = regexp.sub(r'$\1$', line)
1408                 outFile.write(line)
1409             inFile.close()
1410             outFile.close()
1411             # Forcibly overwrite the original file
1412             os.unlink(file)
1413             shutil.move(outFileName, file)
1414         except:
1415             # cleanup our temporary file
1416             os.unlink(outFileName)
1417             print "Failed to strip RCS keywords in %s" % file
1418             raise
1419
1420         print "Patched up RCS keywords in %s" % file
1421
1422     def p4UserForCommit(self,id):
1423         # Return the tuple (perforce user,git email) for a given git commit id
1424         self.getUserMapFromPerforceServer()
1425         gitEmail = read_pipe(["git", "log", "--max-count=1",
1426                               "--format=%ae", id])
1427         gitEmail = gitEmail.strip()
1428         if not self.emails.has_key(gitEmail):
1429             return (None,gitEmail)
1430         else:
1431             return (self.emails[gitEmail],gitEmail)
1432
1433     def checkValidP4Users(self,commits):
1434         # check if any git authors cannot be mapped to p4 users
1435         for id in commits:
1436             (user,email) = self.p4UserForCommit(id)
1437             if not user:
1438                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1439                 if gitConfigBool("git-p4.allowMissingP4Users"):
1440                     print "%s" % msg
1441                 else:
1442                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1443
1444     def lastP4Changelist(self):
1445         # Get back the last changelist number submitted in this client spec. This
1446         # then gets used to patch up the username in the change. If the same
1447         # client spec is being used by multiple processes then this might go
1448         # wrong.
1449         results = p4CmdList("client -o")        # find the current client
1450         client = None
1451         for r in results:
1452             if r.has_key('Client'):
1453                 client = r['Client']
1454                 break
1455         if not client:
1456             die("could not get client spec")
1457         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1458         for r in results:
1459             if r.has_key('change'):
1460                 return r['change']
1461         die("Could not get changelist number for last submit - cannot patch up user details")
1462
1463     def modifyChangelistUser(self, changelist, newUser):
1464         # fixup the user field of a changelist after it has been submitted.
1465         changes = p4CmdList("change -o %s" % changelist)
1466         if len(changes) != 1:
1467             die("Bad output from p4 change modifying %s to user %s" %
1468                 (changelist, newUser))
1469
1470         c = changes[0]
1471         if c['User'] == newUser: return   # nothing to do
1472         c['User'] = newUser
1473         input = marshal.dumps(c)
1474
1475         result = p4CmdList("change -f -i", stdin=input)
1476         for r in result:
1477             if r.has_key('code'):
1478                 if r['code'] == 'error':
1479                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1480             if r.has_key('data'):
1481                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1482                 return
1483         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1484
1485     def canChangeChangelists(self):
1486         # check to see if we have p4 admin or super-user permissions, either of
1487         # which are required to modify changelists.
1488         results = p4CmdList(["protects", self.depotPath])
1489         for r in results:
1490             if r.has_key('perm'):
1491                 if r['perm'] == 'admin':
1492                     return 1
1493                 if r['perm'] == 'super':
1494                     return 1
1495         return 0
1496
1497     def prepareSubmitTemplate(self, changelist=None):
1498         """Run "p4 change -o" to grab a change specification template.
1499            This does not use "p4 -G", as it is nice to keep the submission
1500            template in original order, since a human might edit it.
1501
1502            Remove lines in the Files section that show changes to files
1503            outside the depot path we're committing into."""
1504
1505         [upstream, settings] = findUpstreamBranchPoint()
1506
1507         template = ""
1508         inFilesSection = False
1509         args = ['change', '-o']
1510         if changelist:
1511             args.append(str(changelist))
1512
1513         for line in p4_read_pipe_lines(args):
1514             if line.endswith("\r\n"):
1515                 line = line[:-2] + "\n"
1516             if inFilesSection:
1517                 if line.startswith("\t"):
1518                     # path starts and ends with a tab
1519                     path = line[1:]
1520                     lastTab = path.rfind("\t")
1521                     if lastTab != -1:
1522                         path = path[:lastTab]
1523                         if settings.has_key('depot-paths'):
1524                             if not [p for p in settings['depot-paths']
1525                                     if p4PathStartsWith(path, p)]:
1526                                 continue
1527                         else:
1528                             if not p4PathStartsWith(path, self.depotPath):
1529                                 continue
1530                 else:
1531                     inFilesSection = False
1532             else:
1533                 if line.startswith("Files:"):
1534                     inFilesSection = True
1535
1536             template += line
1537
1538         return template
1539
1540     def edit_template(self, template_file):
1541         """Invoke the editor to let the user change the submission
1542            message.  Return true if okay to continue with the submit."""
1543
1544         # if configured to skip the editing part, just submit
1545         if gitConfigBool("git-p4.skipSubmitEdit"):
1546             return True
1547
1548         # look at the modification time, to check later if the user saved
1549         # the file
1550         mtime = os.stat(template_file).st_mtime
1551
1552         # invoke the editor
1553         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1554             editor = os.environ.get("P4EDITOR")
1555         else:
1556             editor = read_pipe("git var GIT_EDITOR").strip()
1557         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1558
1559         # If the file was not saved, prompt to see if this patch should
1560         # be skipped.  But skip this verification step if configured so.
1561         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1562             return True
1563
1564         # modification time updated means user saved the file
1565         if os.stat(template_file).st_mtime > mtime:
1566             return True
1567
1568         while True:
1569             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1570             if response == 'y':
1571                 return True
1572             if response == 'n':
1573                 return False
1574
1575     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1576         # diff
1577         if os.environ.has_key("P4DIFF"):
1578             del(os.environ["P4DIFF"])
1579         diff = ""
1580         for editedFile in editedFiles:
1581             diff += p4_read_pipe(['diff', '-du',
1582                                   wildcard_encode(editedFile)])
1583
1584         # new file diff
1585         newdiff = ""
1586         for newFile in filesToAdd:
1587             newdiff += "==== new file ====\n"
1588             newdiff += "--- /dev/null\n"
1589             newdiff += "+++ %s\n" % newFile
1590
1591             is_link = os.path.islink(newFile)
1592             expect_link = newFile in symlinks
1593
1594             if is_link and expect_link:
1595                 newdiff += "+%s\n" % os.readlink(newFile)
1596             else:
1597                 f = open(newFile, "r")
1598                 for line in f.readlines():
1599                     newdiff += "+" + line
1600                 f.close()
1601
1602         return (diff + newdiff).replace('\r\n', '\n')
1603
1604     def applyCommit(self, id):
1605         """Apply one commit, return True if it succeeded."""
1606
1607         print "Applying", read_pipe(["git", "show", "-s",
1608                                      "--format=format:%h %s", id])
1609
1610         (p4User, gitEmail) = self.p4UserForCommit(id)
1611
1612         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1613         filesToAdd = set()
1614         filesToChangeType = set()
1615         filesToDelete = set()
1616         editedFiles = set()
1617         pureRenameCopy = set()
1618         symlinks = set()
1619         filesToChangeExecBit = {}
1620         all_files = list()
1621
1622         for line in diff:
1623             diff = parseDiffTreeEntry(line)
1624             modifier = diff['status']
1625             path = diff['src']
1626             all_files.append(path)
1627
1628             if modifier == "M":
1629                 p4_edit(path)
1630                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1631                     filesToChangeExecBit[path] = diff['dst_mode']
1632                 editedFiles.add(path)
1633             elif modifier == "A":
1634                 filesToAdd.add(path)
1635                 filesToChangeExecBit[path] = diff['dst_mode']
1636                 if path in filesToDelete:
1637                     filesToDelete.remove(path)
1638
1639                 dst_mode = int(diff['dst_mode'], 8)
1640                 if dst_mode == 0120000:
1641                     symlinks.add(path)
1642
1643             elif modifier == "D":
1644                 filesToDelete.add(path)
1645                 if path in filesToAdd:
1646                     filesToAdd.remove(path)
1647             elif modifier == "C":
1648                 src, dest = diff['src'], diff['dst']
1649                 p4_integrate(src, dest)
1650                 pureRenameCopy.add(dest)
1651                 if diff['src_sha1'] != diff['dst_sha1']:
1652                     p4_edit(dest)
1653                     pureRenameCopy.discard(dest)
1654                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1655                     p4_edit(dest)
1656                     pureRenameCopy.discard(dest)
1657                     filesToChangeExecBit[dest] = diff['dst_mode']
1658                 if self.isWindows:
1659                     # turn off read-only attribute
1660                     os.chmod(dest, stat.S_IWRITE)
1661                 os.unlink(dest)
1662                 editedFiles.add(dest)
1663             elif modifier == "R":
1664                 src, dest = diff['src'], diff['dst']
1665                 if self.p4HasMoveCommand:
1666                     p4_edit(src)        # src must be open before move
1667                     p4_move(src, dest)  # opens for (move/delete, move/add)
1668                 else:
1669                     p4_integrate(src, dest)
1670                     if diff['src_sha1'] != diff['dst_sha1']:
1671                         p4_edit(dest)
1672                     else:
1673                         pureRenameCopy.add(dest)
1674                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1675                     if not self.p4HasMoveCommand:
1676                         p4_edit(dest)   # with move: already open, writable
1677                     filesToChangeExecBit[dest] = diff['dst_mode']
1678                 if not self.p4HasMoveCommand:
1679                     if self.isWindows:
1680                         os.chmod(dest, stat.S_IWRITE)
1681                     os.unlink(dest)
1682                     filesToDelete.add(src)
1683                 editedFiles.add(dest)
1684             elif modifier == "T":
1685                 filesToChangeType.add(path)
1686             else:
1687                 die("unknown modifier %s for %s" % (modifier, path))
1688
1689         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1690         patchcmd = diffcmd + " | git apply "
1691         tryPatchCmd = patchcmd + "--check -"
1692         applyPatchCmd = patchcmd + "--check --apply -"
1693         patch_succeeded = True
1694
1695         if os.system(tryPatchCmd) != 0:
1696             fixed_rcs_keywords = False
1697             patch_succeeded = False
1698             print "Unfortunately applying the change failed!"
1699
1700             # Patch failed, maybe it's just RCS keyword woes. Look through
1701             # the patch to see if that's possible.
1702             if gitConfigBool("git-p4.attemptRCSCleanup"):
1703                 file = None
1704                 pattern = None
1705                 kwfiles = {}
1706                 for file in editedFiles | filesToDelete:
1707                     # did this file's delta contain RCS keywords?
1708                     pattern = p4_keywords_regexp_for_file(file)
1709
1710                     if pattern:
1711                         # this file is a possibility...look for RCS keywords.
1712                         regexp = re.compile(pattern, re.VERBOSE)
1713                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1714                             if regexp.search(line):
1715                                 if verbose:
1716                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1717                                 kwfiles[file] = pattern
1718                                 break
1719
1720                 for file in kwfiles:
1721                     if verbose:
1722                         print "zapping %s with %s" % (line,pattern)
1723                     # File is being deleted, so not open in p4.  Must
1724                     # disable the read-only bit on windows.
1725                     if self.isWindows and file not in editedFiles:
1726                         os.chmod(file, stat.S_IWRITE)
1727                     self.patchRCSKeywords(file, kwfiles[file])
1728                     fixed_rcs_keywords = True
1729
1730             if fixed_rcs_keywords:
1731                 print "Retrying the patch with RCS keywords cleaned up"
1732                 if os.system(tryPatchCmd) == 0:
1733                     patch_succeeded = True
1734
1735         if not patch_succeeded:
1736             for f in editedFiles:
1737                 p4_revert(f)
1738             return False
1739
1740         #
1741         # Apply the patch for real, and do add/delete/+x handling.
1742         #
1743         system(applyPatchCmd)
1744
1745         for f in filesToChangeType:
1746             p4_edit(f, "-t", "auto")
1747         for f in filesToAdd:
1748             p4_add(f)
1749         for f in filesToDelete:
1750             p4_revert(f)
1751             p4_delete(f)
1752
1753         # Set/clear executable bits
1754         for f in filesToChangeExecBit.keys():
1755             mode = filesToChangeExecBit[f]
1756             setP4ExecBit(f, mode)
1757
1758         if self.update_shelve:
1759             print("all_files = %s" % str(all_files))
1760             p4_reopen_in_change(self.update_shelve, all_files)
1761
1762         #
1763         # Build p4 change description, starting with the contents
1764         # of the git commit message.
1765         #
1766         logMessage = extractLogMessageFromGitCommit(id)
1767         logMessage = logMessage.strip()
1768         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1769
1770         template = self.prepareSubmitTemplate(self.update_shelve)
1771         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1772
1773         if self.preserveUser:
1774            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1775
1776         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1777             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1778             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1779             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1780
1781         separatorLine = "######## everything below this line is just the diff #######\n"
1782         if not self.prepare_p4_only:
1783             submitTemplate += separatorLine
1784             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1785
1786         (handle, fileName) = tempfile.mkstemp()
1787         tmpFile = os.fdopen(handle, "w+b")
1788         if self.isWindows:
1789             submitTemplate = submitTemplate.replace("\n", "\r\n")
1790         tmpFile.write(submitTemplate)
1791         tmpFile.close()
1792
1793         if self.prepare_p4_only:
1794             #
1795             # Leave the p4 tree prepared, and the submit template around
1796             # and let the user decide what to do next
1797             #
1798             print
1799             print "P4 workspace prepared for submission."
1800             print "To submit or revert, go to client workspace"
1801             print "  " + self.clientPath
1802             print
1803             print "To submit, use \"p4 submit\" to write a new description,"
1804             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1805                   " \"git p4\"." % fileName
1806             print "You can delete the file \"%s\" when finished." % fileName
1807
1808             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1809                 print "To preserve change ownership by user %s, you must\n" \
1810                       "do \"p4 change -f <change>\" after submitting and\n" \
1811                       "edit the User field."
1812             if pureRenameCopy:
1813                 print "After submitting, renamed files must be re-synced."
1814                 print "Invoke \"p4 sync -f\" on each of these files:"
1815                 for f in pureRenameCopy:
1816                     print "  " + f
1817
1818             print
1819             print "To revert the changes, use \"p4 revert ...\", and delete"
1820             print "the submit template file \"%s\"" % fileName
1821             if filesToAdd:
1822                 print "Since the commit adds new files, they must be deleted:"
1823                 for f in filesToAdd:
1824                     print "  " + f
1825             print
1826             return True
1827
1828         #
1829         # Let the user edit the change description, then submit it.
1830         #
1831         submitted = False
1832
1833         try:
1834             if self.edit_template(fileName):
1835                 # read the edited message and submit
1836                 tmpFile = open(fileName, "rb")
1837                 message = tmpFile.read()
1838                 tmpFile.close()
1839                 if self.isWindows:
1840                     message = message.replace("\r\n", "\n")
1841                 submitTemplate = message[:message.index(separatorLine)]
1842
1843                 if self.update_shelve:
1844                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1845                 elif self.shelve:
1846                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1847                 else:
1848                     p4_write_pipe(['submit', '-i'], submitTemplate)
1849                     # The rename/copy happened by applying a patch that created a
1850                     # new file.  This leaves it writable, which confuses p4.
1851                     for f in pureRenameCopy:
1852                         p4_sync(f, "-f")
1853
1854                 if self.preserveUser:
1855                     if p4User:
1856                         # Get last changelist number. Cannot easily get it from
1857                         # the submit command output as the output is
1858                         # unmarshalled.
1859                         changelist = self.lastP4Changelist()
1860                         self.modifyChangelistUser(changelist, p4User)
1861
1862                 submitted = True
1863
1864         finally:
1865             # skip this patch
1866             if not submitted or self.shelve:
1867                 if self.shelve:
1868                     print ("Reverting shelved files.")
1869                 else:
1870                     print ("Submission cancelled, undoing p4 changes.")
1871                 for f in editedFiles | filesToDelete:
1872                     p4_revert(f)
1873                 for f in filesToAdd:
1874                     p4_revert(f)
1875                     os.remove(f)
1876
1877         os.remove(fileName)
1878         return submitted
1879
1880     # Export git tags as p4 labels. Create a p4 label and then tag
1881     # with that.
1882     def exportGitTags(self, gitTags):
1883         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1884         if len(validLabelRegexp) == 0:
1885             validLabelRegexp = defaultLabelRegexp
1886         m = re.compile(validLabelRegexp)
1887
1888         for name in gitTags:
1889
1890             if not m.match(name):
1891                 if verbose:
1892                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1893                 continue
1894
1895             # Get the p4 commit this corresponds to
1896             logMessage = extractLogMessageFromGitCommit(name)
1897             values = extractSettingsGitLog(logMessage)
1898
1899             if not values.has_key('change'):
1900                 # a tag pointing to something not sent to p4; ignore
1901                 if verbose:
1902                     print "git tag %s does not give a p4 commit" % name
1903                 continue
1904             else:
1905                 changelist = values['change']
1906
1907             # Get the tag details.
1908             inHeader = True
1909             isAnnotated = False
1910             body = []
1911             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1912                 l = l.strip()
1913                 if inHeader:
1914                     if re.match(r'tag\s+', l):
1915                         isAnnotated = True
1916                     elif re.match(r'\s*$', l):
1917                         inHeader = False
1918                         continue
1919                 else:
1920                     body.append(l)
1921
1922             if not isAnnotated:
1923                 body = ["lightweight tag imported by git p4\n"]
1924
1925             # Create the label - use the same view as the client spec we are using
1926             clientSpec = getClientSpec()
1927
1928             labelTemplate  = "Label: %s\n" % name
1929             labelTemplate += "Description:\n"
1930             for b in body:
1931                 labelTemplate += "\t" + b + "\n"
1932             labelTemplate += "View:\n"
1933             for depot_side in clientSpec.mappings:
1934                 labelTemplate += "\t%s\n" % depot_side
1935
1936             if self.dry_run:
1937                 print "Would create p4 label %s for tag" % name
1938             elif self.prepare_p4_only:
1939                 print "Not creating p4 label %s for tag due to option" \
1940                       " --prepare-p4-only" % name
1941             else:
1942                 p4_write_pipe(["label", "-i"], labelTemplate)
1943
1944                 # Use the label
1945                 p4_system(["tag", "-l", name] +
1946                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1947
1948                 if verbose:
1949                     print "created p4 label for tag %s" % name
1950
1951     def run(self, args):
1952         if len(args) == 0:
1953             self.master = currentGitBranch()
1954         elif len(args) == 1:
1955             self.master = args[0]
1956             if not branchExists(self.master):
1957                 die("Branch %s does not exist" % self.master)
1958         else:
1959             return False
1960
1961         if self.master:
1962             allowSubmit = gitConfig("git-p4.allowSubmit")
1963             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1964                 die("%s is not in git-p4.allowSubmit" % self.master)
1965
1966         [upstream, settings] = findUpstreamBranchPoint()
1967         self.depotPath = settings['depot-paths'][0]
1968         if len(self.origin) == 0:
1969             self.origin = upstream
1970
1971         if self.update_shelve:
1972             self.shelve = True
1973
1974         if self.preserveUser:
1975             if not self.canChangeChangelists():
1976                 die("Cannot preserve user names without p4 super-user or admin permissions")
1977
1978         # if not set from the command line, try the config file
1979         if self.conflict_behavior is None:
1980             val = gitConfig("git-p4.conflict")
1981             if val:
1982                 if val not in self.conflict_behavior_choices:
1983                     die("Invalid value '%s' for config git-p4.conflict" % val)
1984             else:
1985                 val = "ask"
1986             self.conflict_behavior = val
1987
1988         if self.verbose:
1989             print "Origin branch is " + self.origin
1990
1991         if len(self.depotPath) == 0:
1992             print "Internal error: cannot locate perforce depot path from existing branches"
1993             sys.exit(128)
1994
1995         self.useClientSpec = False
1996         if gitConfigBool("git-p4.useclientspec"):
1997             self.useClientSpec = True
1998         if self.useClientSpec:
1999             self.clientSpecDirs = getClientSpec()
2000
2001         # Check for the existence of P4 branches
2002         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2003
2004         if self.useClientSpec and not branchesDetected:
2005             # all files are relative to the client spec
2006             self.clientPath = getClientRoot()
2007         else:
2008             self.clientPath = p4Where(self.depotPath)
2009
2010         if self.clientPath == "":
2011             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2012
2013         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2014         self.oldWorkingDirectory = os.getcwd()
2015
2016         # ensure the clientPath exists
2017         new_client_dir = False
2018         if not os.path.exists(self.clientPath):
2019             new_client_dir = True
2020             os.makedirs(self.clientPath)
2021
2022         chdir(self.clientPath, is_client_path=True)
2023         if self.dry_run:
2024             print "Would synchronize p4 checkout in %s" % self.clientPath
2025         else:
2026             print "Synchronizing p4 checkout..."
2027             if new_client_dir:
2028                 # old one was destroyed, and maybe nobody told p4
2029                 p4_sync("...", "-f")
2030             else:
2031                 p4_sync("...")
2032         self.check()
2033
2034         commits = []
2035         if self.master:
2036             commitish = self.master
2037         else:
2038             commitish = 'HEAD'
2039
2040         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2041             commits.append(line.strip())
2042         commits.reverse()
2043
2044         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2045             self.checkAuthorship = False
2046         else:
2047             self.checkAuthorship = True
2048
2049         if self.preserveUser:
2050             self.checkValidP4Users(commits)
2051
2052         #
2053         # Build up a set of options to be passed to diff when
2054         # submitting each commit to p4.
2055         #
2056         if self.detectRenames:
2057             # command-line -M arg
2058             self.diffOpts = "-M"
2059         else:
2060             # If not explicitly set check the config variable
2061             detectRenames = gitConfig("git-p4.detectRenames")
2062
2063             if detectRenames.lower() == "false" or detectRenames == "":
2064                 self.diffOpts = ""
2065             elif detectRenames.lower() == "true":
2066                 self.diffOpts = "-M"
2067             else:
2068                 self.diffOpts = "-M%s" % detectRenames
2069
2070         # no command-line arg for -C or --find-copies-harder, just
2071         # config variables
2072         detectCopies = gitConfig("git-p4.detectCopies")
2073         if detectCopies.lower() == "false" or detectCopies == "":
2074             pass
2075         elif detectCopies.lower() == "true":
2076             self.diffOpts += " -C"
2077         else:
2078             self.diffOpts += " -C%s" % detectCopies
2079
2080         if gitConfigBool("git-p4.detectCopiesHarder"):
2081             self.diffOpts += " --find-copies-harder"
2082
2083         #
2084         # Apply the commits, one at a time.  On failure, ask if should
2085         # continue to try the rest of the patches, or quit.
2086         #
2087         if self.dry_run:
2088             print "Would apply"
2089         applied = []
2090         last = len(commits) - 1
2091         for i, commit in enumerate(commits):
2092             if self.dry_run:
2093                 print " ", read_pipe(["git", "show", "-s",
2094                                       "--format=format:%h %s", commit])
2095                 ok = True
2096             else:
2097                 ok = self.applyCommit(commit)
2098             if ok:
2099                 applied.append(commit)
2100             else:
2101                 if self.prepare_p4_only and i < last:
2102                     print "Processing only the first commit due to option" \
2103                           " --prepare-p4-only"
2104                     break
2105                 if i < last:
2106                     quit = False
2107                     while True:
2108                         # prompt for what to do, or use the option/variable
2109                         if self.conflict_behavior == "ask":
2110                             print "What do you want to do?"
2111                             response = raw_input("[s]kip this commit but apply"
2112                                                  " the rest, or [q]uit? ")
2113                             if not response:
2114                                 continue
2115                         elif self.conflict_behavior == "skip":
2116                             response = "s"
2117                         elif self.conflict_behavior == "quit":
2118                             response = "q"
2119                         else:
2120                             die("Unknown conflict_behavior '%s'" %
2121                                 self.conflict_behavior)
2122
2123                         if response[0] == "s":
2124                             print "Skipping this commit, but applying the rest"
2125                             break
2126                         if response[0] == "q":
2127                             print "Quitting"
2128                             quit = True
2129                             break
2130                     if quit:
2131                         break
2132
2133         chdir(self.oldWorkingDirectory)
2134         shelved_applied = "shelved" if self.shelve else "applied"
2135         if self.dry_run:
2136             pass
2137         elif self.prepare_p4_only:
2138             pass
2139         elif len(commits) == len(applied):
2140             print ("All commits {0}!".format(shelved_applied))
2141
2142             sync = P4Sync()
2143             if self.branch:
2144                 sync.branch = self.branch
2145             sync.run([])
2146
2147             rebase = P4Rebase()
2148             rebase.rebase()
2149
2150         else:
2151             if len(applied) == 0:
2152                 print ("No commits {0}.".format(shelved_applied))
2153             else:
2154                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2155                 for c in commits:
2156                     if c in applied:
2157                         star = "*"
2158                     else:
2159                         star = " "
2160                     print star, read_pipe(["git", "show", "-s",
2161                                            "--format=format:%h %s",  c])
2162                 print "You will have to do 'git p4 sync' and rebase."
2163
2164         if gitConfigBool("git-p4.exportLabels"):
2165             self.exportLabels = True
2166
2167         if self.exportLabels:
2168             p4Labels = getP4Labels(self.depotPath)
2169             gitTags = getGitTags()
2170
2171             missingGitTags = gitTags - p4Labels
2172             self.exportGitTags(missingGitTags)
2173
2174         # exit with error unless everything applied perfectly
2175         if len(commits) != len(applied):
2176                 sys.exit(1)
2177
2178         return True
2179
2180 class View(object):
2181     """Represent a p4 view ("p4 help views"), and map files in a
2182        repo according to the view."""
2183
2184     def __init__(self, client_name):
2185         self.mappings = []
2186         self.client_prefix = "//%s/" % client_name
2187         # cache results of "p4 where" to lookup client file locations
2188         self.client_spec_path_cache = {}
2189
2190     def append(self, view_line):
2191         """Parse a view line, splitting it into depot and client
2192            sides.  Append to self.mappings, preserving order.  This
2193            is only needed for tag creation."""
2194
2195         # Split the view line into exactly two words.  P4 enforces
2196         # structure on these lines that simplifies this quite a bit.
2197         #
2198         # Either or both words may be double-quoted.
2199         # Single quotes do not matter.
2200         # Double-quote marks cannot occur inside the words.
2201         # A + or - prefix is also inside the quotes.
2202         # There are no quotes unless they contain a space.
2203         # The line is already white-space stripped.
2204         # The two words are separated by a single space.
2205         #
2206         if view_line[0] == '"':
2207             # First word is double quoted.  Find its end.
2208             close_quote_index = view_line.find('"', 1)
2209             if close_quote_index <= 0:
2210                 die("No first-word closing quote found: %s" % view_line)
2211             depot_side = view_line[1:close_quote_index]
2212             # skip closing quote and space
2213             rhs_index = close_quote_index + 1 + 1
2214         else:
2215             space_index = view_line.find(" ")
2216             if space_index <= 0:
2217                 die("No word-splitting space found: %s" % view_line)
2218             depot_side = view_line[0:space_index]
2219             rhs_index = space_index + 1
2220
2221         # prefix + means overlay on previous mapping
2222         if depot_side.startswith("+"):
2223             depot_side = depot_side[1:]
2224
2225         # prefix - means exclude this path, leave out of mappings
2226         exclude = False
2227         if depot_side.startswith("-"):
2228             exclude = True
2229             depot_side = depot_side[1:]
2230
2231         if not exclude:
2232             self.mappings.append(depot_side)
2233
2234     def convert_client_path(self, clientFile):
2235         # chop off //client/ part to make it relative
2236         if not clientFile.startswith(self.client_prefix):
2237             die("No prefix '%s' on clientFile '%s'" %
2238                 (self.client_prefix, clientFile))
2239         return clientFile[len(self.client_prefix):]
2240
2241     def update_client_spec_path_cache(self, files):
2242         """ Caching file paths by "p4 where" batch query """
2243
2244         # List depot file paths exclude that already cached
2245         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2246
2247         if len(fileArgs) == 0:
2248             return  # All files in cache
2249
2250         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2251         for res in where_result:
2252             if "code" in res and res["code"] == "error":
2253                 # assume error is "... file(s) not in client view"
2254                 continue
2255             if "clientFile" not in res:
2256                 die("No clientFile in 'p4 where' output")
2257             if "unmap" in res:
2258                 # it will list all of them, but only one not unmap-ped
2259                 continue
2260             if gitConfigBool("core.ignorecase"):
2261                 res['depotFile'] = res['depotFile'].lower()
2262             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2263
2264         # not found files or unmap files set to ""
2265         for depotFile in fileArgs:
2266             if gitConfigBool("core.ignorecase"):
2267                 depotFile = depotFile.lower()
2268             if depotFile not in self.client_spec_path_cache:
2269                 self.client_spec_path_cache[depotFile] = ""
2270
2271     def map_in_client(self, depot_path):
2272         """Return the relative location in the client where this
2273            depot file should live.  Returns "" if the file should
2274            not be mapped in the client."""
2275
2276         if gitConfigBool("core.ignorecase"):
2277             depot_path = depot_path.lower()
2278
2279         if depot_path in self.client_spec_path_cache:
2280             return self.client_spec_path_cache[depot_path]
2281
2282         die( "Error: %s is not found in client spec path" % depot_path )
2283         return ""
2284
2285 class P4Sync(Command, P4UserMap):
2286     delete_actions = ( "delete", "move/delete", "purge" )
2287
2288     def __init__(self):
2289         Command.__init__(self)
2290         P4UserMap.__init__(self)
2291         self.options = [
2292                 optparse.make_option("--branch", dest="branch"),
2293                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2294                 optparse.make_option("--changesfile", dest="changesFile"),
2295                 optparse.make_option("--silent", dest="silent", action="store_true"),
2296                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2297                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2298                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2299                                      help="Import into refs/heads/ , not refs/remotes"),
2300                 optparse.make_option("--max-changes", dest="maxChanges",
2301                                      help="Maximum number of changes to import"),
2302                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2303                                      help="Internal block size to use when iteratively calling p4 changes"),
2304                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2305                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2306                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2307                                      help="Only sync files that are included in the Perforce Client Spec"),
2308                 optparse.make_option("-/", dest="cloneExclude",
2309                                      action="append", type="string",
2310                                      help="exclude depot path"),
2311         ]
2312         self.description = """Imports from Perforce into a git repository.\n
2313     example:
2314     //depot/my/project/ -- to import the current head
2315     //depot/my/project/@all -- to import everything
2316     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2317
2318     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2319
2320         self.usage += " //depot/path[@revRange]"
2321         self.silent = False
2322         self.createdBranches = set()
2323         self.committedChanges = set()
2324         self.branch = ""
2325         self.detectBranches = False
2326         self.detectLabels = False
2327         self.importLabels = False
2328         self.changesFile = ""
2329         self.syncWithOrigin = True
2330         self.importIntoRemotes = True
2331         self.maxChanges = ""
2332         self.changes_block_size = None
2333         self.keepRepoPath = False
2334         self.depotPaths = None
2335         self.p4BranchesInGit = []
2336         self.cloneExclude = []
2337         self.useClientSpec = False
2338         self.useClientSpec_from_options = False
2339         self.clientSpecDirs = None
2340         self.tempBranches = []
2341         self.tempBranchLocation = "refs/git-p4-tmp"
2342         self.largeFileSystem = None
2343
2344         if gitConfig('git-p4.largeFileSystem'):
2345             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2346             self.largeFileSystem = largeFileSystemConstructor(
2347                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2348             )
2349
2350         if gitConfig("git-p4.syncFromOrigin") == "false":
2351             self.syncWithOrigin = False
2352
2353     # This is required for the "append" cloneExclude action
2354     def ensure_value(self, attr, value):
2355         if not hasattr(self, attr) or getattr(self, attr) is None:
2356             setattr(self, attr, value)
2357         return getattr(self, attr)
2358
2359     # Force a checkpoint in fast-import and wait for it to finish
2360     def checkpoint(self):
2361         self.gitStream.write("checkpoint\n\n")
2362         self.gitStream.write("progress checkpoint\n\n")
2363         out = self.gitOutput.readline()
2364         if self.verbose:
2365             print "checkpoint finished: " + out
2366
2367     def extractFilesFromCommit(self, commit):
2368         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2369                              for path in self.cloneExclude]
2370         files = []
2371         fnum = 0
2372         while commit.has_key("depotFile%s" % fnum):
2373             path =  commit["depotFile%s" % fnum]
2374
2375             if [p for p in self.cloneExclude
2376                 if p4PathStartsWith(path, p)]:
2377                 found = False
2378             else:
2379                 found = [p for p in self.depotPaths
2380                          if p4PathStartsWith(path, p)]
2381             if not found:
2382                 fnum = fnum + 1
2383                 continue
2384
2385             file = {}
2386             file["path"] = path
2387             file["rev"] = commit["rev%s" % fnum]
2388             file["action"] = commit["action%s" % fnum]
2389             file["type"] = commit["type%s" % fnum]
2390             files.append(file)
2391             fnum = fnum + 1
2392         return files
2393
2394     def extractJobsFromCommit(self, commit):
2395         jobs = []
2396         jnum = 0
2397         while commit.has_key("job%s" % jnum):
2398             job = commit["job%s" % jnum]
2399             jobs.append(job)
2400             jnum = jnum + 1
2401         return jobs
2402
2403     def stripRepoPath(self, path, prefixes):
2404         """When streaming files, this is called to map a p4 depot path
2405            to where it should go in git.  The prefixes are either
2406            self.depotPaths, or self.branchPrefixes in the case of
2407            branch detection."""
2408
2409         if self.useClientSpec:
2410             # branch detection moves files up a level (the branch name)
2411             # from what client spec interpretation gives
2412             path = self.clientSpecDirs.map_in_client(path)
2413             if self.detectBranches:
2414                 for b in self.knownBranches:
2415                     if path.startswith(b + "/"):
2416                         path = path[len(b)+1:]
2417
2418         elif self.keepRepoPath:
2419             # Preserve everything in relative path name except leading
2420             # //depot/; just look at first prefix as they all should
2421             # be in the same depot.
2422             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2423             if p4PathStartsWith(path, depot):
2424                 path = path[len(depot):]
2425
2426         else:
2427             for p in prefixes:
2428                 if p4PathStartsWith(path, p):
2429                     path = path[len(p):]
2430                     break
2431
2432         path = wildcard_decode(path)
2433         return path
2434
2435     def splitFilesIntoBranches(self, commit):
2436         """Look at each depotFile in the commit to figure out to what
2437            branch it belongs."""
2438
2439         if self.clientSpecDirs:
2440             files = self.extractFilesFromCommit(commit)
2441             self.clientSpecDirs.update_client_spec_path_cache(files)
2442
2443         branches = {}
2444         fnum = 0
2445         while commit.has_key("depotFile%s" % fnum):
2446             path =  commit["depotFile%s" % fnum]
2447             found = [p for p in self.depotPaths
2448                      if p4PathStartsWith(path, p)]
2449             if not found:
2450                 fnum = fnum + 1
2451                 continue
2452
2453             file = {}
2454             file["path"] = path
2455             file["rev"] = commit["rev%s" % fnum]
2456             file["action"] = commit["action%s" % fnum]
2457             file["type"] = commit["type%s" % fnum]
2458             fnum = fnum + 1
2459
2460             # start with the full relative path where this file would
2461             # go in a p4 client
2462             if self.useClientSpec:
2463                 relPath = self.clientSpecDirs.map_in_client(path)
2464             else:
2465                 relPath = self.stripRepoPath(path, self.depotPaths)
2466
2467             for branch in self.knownBranches.keys():
2468                 # add a trailing slash so that a commit into qt/4.2foo
2469                 # doesn't end up in qt/4.2, e.g.
2470                 if relPath.startswith(branch + "/"):
2471                     if branch not in branches:
2472                         branches[branch] = []
2473                     branches[branch].append(file)
2474                     break
2475
2476         return branches
2477
2478     def writeToGitStream(self, gitMode, relPath, contents):
2479         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2480         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2481         for d in contents:
2482             self.gitStream.write(d)
2483         self.gitStream.write('\n')
2484
2485     # output one file from the P4 stream
2486     # - helper for streamP4Files
2487
2488     def streamOneP4File(self, file, contents):
2489         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2490         if verbose:
2491             size = int(self.stream_file['fileSize'])
2492             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2493             sys.stdout.flush()
2494
2495         (type_base, type_mods) = split_p4_type(file["type"])
2496
2497         git_mode = "100644"
2498         if "x" in type_mods:
2499             git_mode = "100755"
2500         if type_base == "symlink":
2501             git_mode = "120000"
2502             # p4 print on a symlink sometimes contains "target\n";
2503             # if it does, remove the newline
2504             data = ''.join(contents)
2505             if not data:
2506                 # Some version of p4 allowed creating a symlink that pointed
2507                 # to nothing.  This causes p4 errors when checking out such
2508                 # a change, and errors here too.  Work around it by ignoring
2509                 # the bad symlink; hopefully a future change fixes it.
2510                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2511                 return
2512             elif data[-1] == '\n':
2513                 contents = [data[:-1]]
2514             else:
2515                 contents = [data]
2516
2517         if type_base == "utf16":
2518             # p4 delivers different text in the python output to -G
2519             # than it does when using "print -o", or normal p4 client
2520             # operations.  utf16 is converted to ascii or utf8, perhaps.
2521             # But ascii text saved as -t utf16 is completely mangled.
2522             # Invoke print -o to get the real contents.
2523             #
2524             # On windows, the newlines will always be mangled by print, so put
2525             # them back too.  This is not needed to the cygwin windows version,
2526             # just the native "NT" type.
2527             #
2528             try:
2529                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2530             except Exception as e:
2531                 if 'Translation of file content failed' in str(e):
2532                     type_base = 'binary'
2533                 else:
2534                     raise e
2535             else:
2536                 if p4_version_string().find('/NT') >= 0:
2537                     text = text.replace('\r\n', '\n')
2538                 contents = [ text ]
2539
2540         if type_base == "apple":
2541             # Apple filetype files will be streamed as a concatenation of
2542             # its appledouble header and the contents.  This is useless
2543             # on both macs and non-macs.  If using "print -q -o xx", it
2544             # will create "xx" with the data, and "%xx" with the header.
2545             # This is also not very useful.
2546             #
2547             # Ideally, someday, this script can learn how to generate
2548             # appledouble files directly and import those to git, but
2549             # non-mac machines can never find a use for apple filetype.
2550             print "\nIgnoring apple filetype file %s" % file['depotFile']
2551             return
2552
2553         # Note that we do not try to de-mangle keywords on utf16 files,
2554         # even though in theory somebody may want that.
2555         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2556         if pattern:
2557             regexp = re.compile(pattern, re.VERBOSE)
2558             text = ''.join(contents)
2559             text = regexp.sub(r'$\1$', text)
2560             contents = [ text ]
2561
2562         try:
2563             relPath.decode('ascii')
2564         except:
2565             encoding = 'utf8'
2566             if gitConfig('git-p4.pathEncoding'):
2567                 encoding = gitConfig('git-p4.pathEncoding')
2568             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2569             if self.verbose:
2570                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2571
2572         if self.largeFileSystem:
2573             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2574
2575         self.writeToGitStream(git_mode, relPath, contents)
2576
2577     def streamOneP4Deletion(self, file):
2578         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2579         if verbose:
2580             sys.stdout.write("delete %s\n" % relPath)
2581             sys.stdout.flush()
2582         self.gitStream.write("D %s\n" % relPath)
2583
2584         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2585             self.largeFileSystem.removeLargeFile(relPath)
2586
2587     # handle another chunk of streaming data
2588     def streamP4FilesCb(self, marshalled):
2589
2590         # catch p4 errors and complain
2591         err = None
2592         if "code" in marshalled:
2593             if marshalled["code"] == "error":
2594                 if "data" in marshalled:
2595                     err = marshalled["data"].rstrip()
2596
2597         if not err and 'fileSize' in self.stream_file:
2598             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2599             if required_bytes > 0:
2600                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2601                     os.getcwd(), required_bytes/1024/1024
2602                 )
2603
2604         if err:
2605             f = None
2606             if self.stream_have_file_info:
2607                 if "depotFile" in self.stream_file:
2608                     f = self.stream_file["depotFile"]
2609             # force a failure in fast-import, else an empty
2610             # commit will be made
2611             self.gitStream.write("\n")
2612             self.gitStream.write("die-now\n")
2613             self.gitStream.close()
2614             # ignore errors, but make sure it exits first
2615             self.importProcess.wait()
2616             if f:
2617                 die("Error from p4 print for %s: %s" % (f, err))
2618             else:
2619                 die("Error from p4 print: %s" % err)
2620
2621         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2622             # start of a new file - output the old one first
2623             self.streamOneP4File(self.stream_file, self.stream_contents)
2624             self.stream_file = {}
2625             self.stream_contents = []
2626             self.stream_have_file_info = False
2627
2628         # pick up the new file information... for the
2629         # 'data' field we need to append to our array
2630         for k in marshalled.keys():
2631             if k == 'data':
2632                 if 'streamContentSize' not in self.stream_file:
2633                     self.stream_file['streamContentSize'] = 0
2634                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2635                 self.stream_contents.append(marshalled['data'])
2636             else:
2637                 self.stream_file[k] = marshalled[k]
2638
2639         if (verbose and
2640             'streamContentSize' in self.stream_file and
2641             'fileSize' in self.stream_file and
2642             'depotFile' in self.stream_file):
2643             size = int(self.stream_file["fileSize"])
2644             if size > 0:
2645                 progress = 100*self.stream_file['streamContentSize']/size
2646                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2647                 sys.stdout.flush()
2648
2649         self.stream_have_file_info = True
2650
2651     # Stream directly from "p4 files" into "git fast-import"
2652     def streamP4Files(self, files):
2653         filesForCommit = []
2654         filesToRead = []
2655         filesToDelete = []
2656
2657         for f in files:
2658             filesForCommit.append(f)
2659             if f['action'] in self.delete_actions:
2660                 filesToDelete.append(f)
2661             else:
2662                 filesToRead.append(f)
2663
2664         # deleted files...
2665         for f in filesToDelete:
2666             self.streamOneP4Deletion(f)
2667
2668         if len(filesToRead) > 0:
2669             self.stream_file = {}
2670             self.stream_contents = []
2671             self.stream_have_file_info = False
2672
2673             # curry self argument
2674             def streamP4FilesCbSelf(entry):
2675                 self.streamP4FilesCb(entry)
2676
2677             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2678
2679             p4CmdList(["-x", "-", "print"],
2680                       stdin=fileArgs,
2681                       cb=streamP4FilesCbSelf)
2682
2683             # do the last chunk
2684             if self.stream_file.has_key('depotFile'):
2685                 self.streamOneP4File(self.stream_file, self.stream_contents)
2686
2687     def make_email(self, userid):
2688         if userid in self.users:
2689             return self.users[userid]
2690         else:
2691             return "%s <a@b>" % userid
2692
2693     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2694         """ Stream a p4 tag.
2695         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2696         """
2697
2698         if verbose:
2699             print "writing tag %s for commit %s" % (labelName, commit)
2700         gitStream.write("tag %s\n" % labelName)
2701         gitStream.write("from %s\n" % commit)
2702
2703         if labelDetails.has_key('Owner'):
2704             owner = labelDetails["Owner"]
2705         else:
2706             owner = None
2707
2708         # Try to use the owner of the p4 label, or failing that,
2709         # the current p4 user id.
2710         if owner:
2711             email = self.make_email(owner)
2712         else:
2713             email = self.make_email(self.p4UserId())
2714         tagger = "%s %s %s" % (email, epoch, self.tz)
2715
2716         gitStream.write("tagger %s\n" % tagger)
2717
2718         print "labelDetails=",labelDetails
2719         if labelDetails.has_key('Description'):
2720             description = labelDetails['Description']
2721         else:
2722             description = 'Label from git p4'
2723
2724         gitStream.write("data %d\n" % len(description))
2725         gitStream.write(description)
2726         gitStream.write("\n")
2727
2728     def inClientSpec(self, path):
2729         if not self.clientSpecDirs:
2730             return True
2731         inClientSpec = self.clientSpecDirs.map_in_client(path)
2732         if not inClientSpec and self.verbose:
2733             print('Ignoring file outside of client spec: {0}'.format(path))
2734         return inClientSpec
2735
2736     def hasBranchPrefix(self, path):
2737         if not self.branchPrefixes:
2738             return True
2739         hasPrefix = [p for p in self.branchPrefixes
2740                         if p4PathStartsWith(path, p)]
2741         if not hasPrefix and self.verbose:
2742             print('Ignoring file outside of prefix: {0}'.format(path))
2743         return hasPrefix
2744
2745     def commit(self, details, files, branch, parent = ""):
2746         epoch = details["time"]
2747         author = details["user"]
2748         jobs = self.extractJobsFromCommit(details)
2749
2750         if self.verbose:
2751             print('commit into {0}'.format(branch))
2752
2753         if self.clientSpecDirs:
2754             self.clientSpecDirs.update_client_spec_path_cache(files)
2755
2756         files = [f for f in files
2757             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2758
2759         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2760             print('Ignoring revision {0} as it would produce an empty commit.'
2761                 .format(details['change']))
2762             return
2763
2764         self.gitStream.write("commit %s\n" % branch)
2765         self.gitStream.write("mark :%s\n" % details["change"])
2766         self.committedChanges.add(int(details["change"]))
2767         committer = ""
2768         if author not in self.users:
2769             self.getUserMapFromPerforceServer()
2770         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2771
2772         self.gitStream.write("committer %s\n" % committer)
2773
2774         self.gitStream.write("data <<EOT\n")
2775         self.gitStream.write(details["desc"])
2776         if len(jobs) > 0:
2777             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2778         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2779                              (','.join(self.branchPrefixes), details["change"]))
2780         if len(details['options']) > 0:
2781             self.gitStream.write(": options = %s" % details['options'])
2782         self.gitStream.write("]\nEOT\n\n")
2783
2784         if len(parent) > 0:
2785             if self.verbose:
2786                 print "parent %s" % parent
2787             self.gitStream.write("from %s\n" % parent)
2788
2789         self.streamP4Files(files)
2790         self.gitStream.write("\n")
2791
2792         change = int(details["change"])
2793
2794         if self.labels.has_key(change):
2795             label = self.labels[change]
2796             labelDetails = label[0]
2797             labelRevisions = label[1]
2798             if self.verbose:
2799                 print "Change %s is labelled %s" % (change, labelDetails)
2800
2801             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2802                                                 for p in self.branchPrefixes])
2803
2804             if len(files) == len(labelRevisions):
2805
2806                 cleanedFiles = {}
2807                 for info in files:
2808                     if info["action"] in self.delete_actions:
2809                         continue
2810                     cleanedFiles[info["depotFile"]] = info["rev"]
2811
2812                 if cleanedFiles == labelRevisions:
2813                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2814
2815                 else:
2816                     if not self.silent:
2817                         print ("Tag %s does not match with change %s: files do not match."
2818                                % (labelDetails["label"], change))
2819
2820             else:
2821                 if not self.silent:
2822                     print ("Tag %s does not match with change %s: file count is different."
2823                            % (labelDetails["label"], change))
2824
2825     # Build a dictionary of changelists and labels, for "detect-labels" option.
2826     def getLabels(self):
2827         self.labels = {}
2828
2829         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2830         if len(l) > 0 and not self.silent:
2831             print "Finding files belonging to labels in %s" % `self.depotPaths`
2832
2833         for output in l:
2834             label = output["label"]
2835             revisions = {}
2836             newestChange = 0
2837             if self.verbose:
2838                 print "Querying files for label %s" % label
2839             for file in p4CmdList(["files"] +
2840                                       ["%s...@%s" % (p, label)
2841                                           for p in self.depotPaths]):
2842                 revisions[file["depotFile"]] = file["rev"]
2843                 change = int(file["change"])
2844                 if change > newestChange:
2845                     newestChange = change
2846
2847             self.labels[newestChange] = [output, revisions]
2848
2849         if self.verbose:
2850             print "Label changes: %s" % self.labels.keys()
2851
2852     # Import p4 labels as git tags. A direct mapping does not
2853     # exist, so assume that if all the files are at the same revision
2854     # then we can use that, or it's something more complicated we should
2855     # just ignore.
2856     def importP4Labels(self, stream, p4Labels):
2857         if verbose:
2858             print "import p4 labels: " + ' '.join(p4Labels)
2859
2860         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2861         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2862         if len(validLabelRegexp) == 0:
2863             validLabelRegexp = defaultLabelRegexp
2864         m = re.compile(validLabelRegexp)
2865
2866         for name in p4Labels:
2867             commitFound = False
2868
2869             if not m.match(name):
2870                 if verbose:
2871                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2872                 continue
2873
2874             if name in ignoredP4Labels:
2875                 continue
2876
2877             labelDetails = p4CmdList(['label', "-o", name])[0]
2878
2879             # get the most recent changelist for each file in this label
2880             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2881                                 for p in self.depotPaths])
2882
2883             if change.has_key('change'):
2884                 # find the corresponding git commit; take the oldest commit
2885                 changelist = int(change['change'])
2886                 if changelist in self.committedChanges:
2887                     gitCommit = ":%d" % changelist       # use a fast-import mark
2888                     commitFound = True
2889                 else:
2890                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2891                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2892                     if len(gitCommit) == 0:
2893                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2894                     else:
2895                         commitFound = True
2896                         gitCommit = gitCommit.strip()
2897
2898                 if commitFound:
2899                     # Convert from p4 time format
2900                     try:
2901                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2902                     except ValueError:
2903                         print "Could not convert label time %s" % labelDetails['Update']
2904                         tmwhen = 1
2905
2906                     when = int(time.mktime(tmwhen))
2907                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2908                     if verbose:
2909                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2910             else:
2911                 if verbose:
2912                     print "Label %s has no changelists - possibly deleted?" % name
2913
2914             if not commitFound:
2915                 # We can't import this label; don't try again as it will get very
2916                 # expensive repeatedly fetching all the files for labels that will
2917                 # never be imported. If the label is moved in the future, the
2918                 # ignore will need to be removed manually.
2919                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2920
2921     def guessProjectName(self):
2922         for p in self.depotPaths:
2923             if p.endswith("/"):
2924                 p = p[:-1]
2925             p = p[p.strip().rfind("/") + 1:]
2926             if not p.endswith("/"):
2927                p += "/"
2928             return p
2929
2930     def getBranchMapping(self):
2931         lostAndFoundBranches = set()
2932
2933         user = gitConfig("git-p4.branchUser")
2934         if len(user) > 0:
2935             command = "branches -u %s" % user
2936         else:
2937             command = "branches"
2938
2939         for info in p4CmdList(command):
2940             details = p4Cmd(["branch", "-o", info["branch"]])
2941             viewIdx = 0
2942             while details.has_key("View%s" % viewIdx):
2943                 paths = details["View%s" % viewIdx].split(" ")
2944                 viewIdx = viewIdx + 1
2945                 # require standard //depot/foo/... //depot/bar/... mapping
2946                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2947                     continue
2948                 source = paths[0]
2949                 destination = paths[1]
2950                 ## HACK
2951                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2952                     source = source[len(self.depotPaths[0]):-4]
2953                     destination = destination[len(self.depotPaths[0]):-4]
2954
2955                     if destination in self.knownBranches:
2956                         if not self.silent:
2957                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2958                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2959                         continue
2960
2961                     self.knownBranches[destination] = source
2962
2963                     lostAndFoundBranches.discard(destination)
2964
2965                     if source not in self.knownBranches:
2966                         lostAndFoundBranches.add(source)
2967
2968         # Perforce does not strictly require branches to be defined, so we also
2969         # check git config for a branch list.
2970         #
2971         # Example of branch definition in git config file:
2972         # [git-p4]
2973         #   branchList=main:branchA
2974         #   branchList=main:branchB
2975         #   branchList=branchA:branchC
2976         configBranches = gitConfigList("git-p4.branchList")
2977         for branch in configBranches:
2978             if branch:
2979                 (source, destination) = branch.split(":")
2980                 self.knownBranches[destination] = source
2981
2982                 lostAndFoundBranches.discard(destination)
2983
2984                 if source not in self.knownBranches:
2985                     lostAndFoundBranches.add(source)
2986
2987
2988         for branch in lostAndFoundBranches:
2989             self.knownBranches[branch] = branch
2990
2991     def getBranchMappingFromGitBranches(self):
2992         branches = p4BranchesInGit(self.importIntoRemotes)
2993         for branch in branches.keys():
2994             if branch == "master":
2995                 branch = "main"
2996             else:
2997                 branch = branch[len(self.projectName):]
2998             self.knownBranches[branch] = branch
2999
3000     def updateOptionDict(self, d):
3001         option_keys = {}
3002         if self.keepRepoPath:
3003             option_keys['keepRepoPath'] = 1
3004
3005         d["options"] = ' '.join(sorted(option_keys.keys()))
3006
3007     def readOptions(self, d):
3008         self.keepRepoPath = (d.has_key('options')
3009                              and ('keepRepoPath' in d['options']))
3010
3011     def gitRefForBranch(self, branch):
3012         if branch == "main":
3013             return self.refPrefix + "master"
3014
3015         if len(branch) <= 0:
3016             return branch
3017
3018         return self.refPrefix + self.projectName + branch
3019
3020     def gitCommitByP4Change(self, ref, change):
3021         if self.verbose:
3022             print "looking in ref " + ref + " for change %s using bisect..." % change
3023
3024         earliestCommit = ""
3025         latestCommit = parseRevision(ref)
3026
3027         while True:
3028             if self.verbose:
3029                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3030             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3031             if len(next) == 0:
3032                 if self.verbose:
3033                     print "argh"
3034                 return ""
3035             log = extractLogMessageFromGitCommit(next)
3036             settings = extractSettingsGitLog(log)
3037             currentChange = int(settings['change'])
3038             if self.verbose:
3039                 print "current change %s" % currentChange
3040
3041             if currentChange == change:
3042                 if self.verbose:
3043                     print "found %s" % next
3044                 return next
3045
3046             if currentChange < change:
3047                 earliestCommit = "^%s" % next
3048             else:
3049                 latestCommit = "%s" % next
3050
3051         return ""
3052
3053     def importNewBranch(self, branch, maxChange):
3054         # make fast-import flush all changes to disk and update the refs using the checkpoint
3055         # command so that we can try to find the branch parent in the git history
3056         self.gitStream.write("checkpoint\n\n");
3057         self.gitStream.flush();
3058         branchPrefix = self.depotPaths[0] + branch + "/"
3059         range = "@1,%s" % maxChange
3060         #print "prefix" + branchPrefix
3061         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3062         if len(changes) <= 0:
3063             return False
3064         firstChange = changes[0]
3065         #print "first change in branch: %s" % firstChange
3066         sourceBranch = self.knownBranches[branch]
3067         sourceDepotPath = self.depotPaths[0] + sourceBranch
3068         sourceRef = self.gitRefForBranch(sourceBranch)
3069         #print "source " + sourceBranch
3070
3071         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3072         #print "branch parent: %s" % branchParentChange
3073         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3074         if len(gitParent) > 0:
3075             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3076             #print "parent git commit: %s" % gitParent
3077
3078         self.importChanges(changes)
3079         return True
3080
3081     def searchParent(self, parent, branch, target):
3082         parentFound = False
3083         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3084                                      "--no-merges", parent]):
3085             blob = blob.strip()
3086             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3087                 parentFound = True
3088                 if self.verbose:
3089                     print "Found parent of %s in commit %s" % (branch, blob)
3090                 break
3091         if parentFound:
3092             return blob
3093         else:
3094             return None
3095
3096     def importChanges(self, changes):
3097         cnt = 1
3098         for change in changes:
3099             description = p4_describe(change)
3100             self.updateOptionDict(description)
3101
3102             if not self.silent:
3103                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3104                 sys.stdout.flush()
3105             cnt = cnt + 1
3106
3107             try:
3108                 if self.detectBranches:
3109                     branches = self.splitFilesIntoBranches(description)
3110                     for branch in branches.keys():
3111                         ## HACK  --hwn
3112                         branchPrefix = self.depotPaths[0] + branch + "/"
3113                         self.branchPrefixes = [ branchPrefix ]
3114
3115                         parent = ""
3116
3117                         filesForCommit = branches[branch]
3118
3119                         if self.verbose:
3120                             print "branch is %s" % branch
3121
3122                         self.updatedBranches.add(branch)
3123
3124                         if branch not in self.createdBranches:
3125                             self.createdBranches.add(branch)
3126                             parent = self.knownBranches[branch]
3127                             if parent == branch:
3128                                 parent = ""
3129                             else:
3130                                 fullBranch = self.projectName + branch
3131                                 if fullBranch not in self.p4BranchesInGit:
3132                                     if not self.silent:
3133                                         print("\n    Importing new branch %s" % fullBranch);
3134                                     if self.importNewBranch(branch, change - 1):
3135                                         parent = ""
3136                                         self.p4BranchesInGit.append(fullBranch)
3137                                     if not self.silent:
3138                                         print("\n    Resuming with change %s" % change);
3139
3140                                 if self.verbose:
3141                                     print "parent determined through known branches: %s" % parent
3142
3143                         branch = self.gitRefForBranch(branch)
3144                         parent = self.gitRefForBranch(parent)
3145
3146                         if self.verbose:
3147                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3148
3149                         if len(parent) == 0 and branch in self.initialParents:
3150                             parent = self.initialParents[branch]
3151                             del self.initialParents[branch]
3152
3153                         blob = None
3154                         if len(parent) > 0:
3155                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3156                             if self.verbose:
3157                                 print "Creating temporary branch: " + tempBranch
3158                             self.commit(description, filesForCommit, tempBranch)
3159                             self.tempBranches.append(tempBranch)
3160                             self.checkpoint()
3161                             blob = self.searchParent(parent, branch, tempBranch)
3162                         if blob:
3163                             self.commit(description, filesForCommit, branch, blob)
3164                         else:
3165                             if self.verbose:
3166                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3167                             self.commit(description, filesForCommit, branch, parent)
3168                 else:
3169                     files = self.extractFilesFromCommit(description)
3170                     self.commit(description, files, self.branch,
3171                                 self.initialParent)
3172                     # only needed once, to connect to the previous commit
3173                     self.initialParent = ""
3174             except IOError:
3175                 print self.gitError.read()
3176                 sys.exit(1)
3177
3178     def importHeadRevision(self, revision):
3179         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3180
3181         details = {}
3182         details["user"] = "git perforce import user"
3183         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3184                            % (' '.join(self.depotPaths), revision))
3185         details["change"] = revision
3186         newestRevision = 0
3187
3188         fileCnt = 0
3189         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3190
3191         for info in p4CmdList(["files"] + fileArgs):
3192
3193             if 'code' in info and info['code'] == 'error':
3194                 sys.stderr.write("p4 returned an error: %s\n"
3195                                  % info['data'])
3196                 if info['data'].find("must refer to client") >= 0:
3197                     sys.stderr.write("This particular p4 error is misleading.\n")
3198                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3199                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3200                 sys.exit(1)
3201             if 'p4ExitCode' in info:
3202                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3203                 sys.exit(1)
3204
3205
3206             change = int(info["change"])
3207             if change > newestRevision:
3208                 newestRevision = change
3209
3210             if info["action"] in self.delete_actions:
3211                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3212                 #fileCnt = fileCnt + 1
3213                 continue
3214
3215             for prop in ["depotFile", "rev", "action", "type" ]:
3216                 details["%s%s" % (prop, fileCnt)] = info[prop]
3217
3218             fileCnt = fileCnt + 1
3219
3220         details["change"] = newestRevision
3221
3222         # Use time from top-most change so that all git p4 clones of
3223         # the same p4 repo have the same commit SHA1s.
3224         res = p4_describe(newestRevision)
3225         details["time"] = res["time"]
3226
3227         self.updateOptionDict(details)
3228         try:
3229             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3230         except IOError:
3231             print "IO error with git fast-import. Is your git version recent enough?"
3232             print self.gitError.read()
3233
3234
3235     def run(self, args):
3236         self.depotPaths = []
3237         self.changeRange = ""
3238         self.previousDepotPaths = []
3239         self.hasOrigin = False
3240
3241         # map from branch depot path to parent branch
3242         self.knownBranches = {}
3243         self.initialParents = {}
3244
3245         if self.importIntoRemotes:
3246             self.refPrefix = "refs/remotes/p4/"
3247         else:
3248             self.refPrefix = "refs/heads/p4/"
3249
3250         if self.syncWithOrigin:
3251             self.hasOrigin = originP4BranchesExist()
3252             if self.hasOrigin:
3253                 if not self.silent:
3254                     print 'Syncing with origin first, using "git fetch origin"'
3255                 system("git fetch origin")
3256
3257         branch_arg_given = bool(self.branch)
3258         if len(self.branch) == 0:
3259             self.branch = self.refPrefix + "master"
3260             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3261                 system("git update-ref %s refs/heads/p4" % self.branch)
3262                 system("git branch -D p4")
3263
3264         # accept either the command-line option, or the configuration variable
3265         if self.useClientSpec:
3266             # will use this after clone to set the variable
3267             self.useClientSpec_from_options = True
3268         else:
3269             if gitConfigBool("git-p4.useclientspec"):
3270                 self.useClientSpec = True
3271         if self.useClientSpec:
3272             self.clientSpecDirs = getClientSpec()
3273
3274         # TODO: should always look at previous commits,
3275         # merge with previous imports, if possible.
3276         if args == []:
3277             if self.hasOrigin:
3278                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3279
3280             # branches holds mapping from branch name to sha1
3281             branches = p4BranchesInGit(self.importIntoRemotes)
3282
3283             # restrict to just this one, disabling detect-branches
3284             if branch_arg_given:
3285                 short = self.branch.split("/")[-1]
3286                 if short in branches:
3287                     self.p4BranchesInGit = [ short ]
3288             else:
3289                 self.p4BranchesInGit = branches.keys()
3290
3291             if len(self.p4BranchesInGit) > 1:
3292                 if not self.silent:
3293                     print "Importing from/into multiple branches"
3294                 self.detectBranches = True
3295                 for branch in branches.keys():
3296                     self.initialParents[self.refPrefix + branch] = \
3297                         branches[branch]
3298
3299             if self.verbose:
3300                 print "branches: %s" % self.p4BranchesInGit
3301
3302             p4Change = 0
3303             for branch in self.p4BranchesInGit:
3304                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3305
3306                 settings = extractSettingsGitLog(logMsg)
3307
3308                 self.readOptions(settings)
3309                 if (settings.has_key('depot-paths')
3310                     and settings.has_key ('change')):
3311                     change = int(settings['change']) + 1
3312                     p4Change = max(p4Change, change)
3313
3314                     depotPaths = sorted(settings['depot-paths'])
3315                     if self.previousDepotPaths == []:
3316                         self.previousDepotPaths = depotPaths
3317                     else:
3318                         paths = []
3319                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3320                             prev_list = prev.split("/")
3321                             cur_list = cur.split("/")
3322                             for i in range(0, min(len(cur_list), len(prev_list))):
3323                                 if cur_list[i] <> prev_list[i]:
3324                                     i = i - 1
3325                                     break
3326
3327                             paths.append ("/".join(cur_list[:i + 1]))
3328
3329                         self.previousDepotPaths = paths
3330
3331             if p4Change > 0:
3332                 self.depotPaths = sorted(self.previousDepotPaths)
3333                 self.changeRange = "@%s,#head" % p4Change
3334                 if not self.silent and not self.detectBranches:
3335                     print "Performing incremental import into %s git branch" % self.branch
3336
3337         # accept multiple ref name abbreviations:
3338         #    refs/foo/bar/branch -> use it exactly
3339         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3340         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3341         if not self.branch.startswith("refs/"):
3342             if self.importIntoRemotes:
3343                 prepend = "refs/remotes/"
3344             else:
3345                 prepend = "refs/heads/"
3346             if not self.branch.startswith("p4/"):
3347                 prepend += "p4/"
3348             self.branch = prepend + self.branch
3349
3350         if len(args) == 0 and self.depotPaths:
3351             if not self.silent:
3352                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3353         else:
3354             if self.depotPaths and self.depotPaths != args:
3355                 print ("previous import used depot path %s and now %s was specified. "
3356                        "This doesn't work!" % (' '.join (self.depotPaths),
3357                                                ' '.join (args)))
3358                 sys.exit(1)
3359
3360             self.depotPaths = sorted(args)
3361
3362         revision = ""
3363         self.users = {}
3364
3365         # Make sure no revision specifiers are used when --changesfile
3366         # is specified.
3367         bad_changesfile = False
3368         if len(self.changesFile) > 0:
3369             for p in self.depotPaths:
3370                 if p.find("@") >= 0 or p.find("#") >= 0:
3371                     bad_changesfile = True
3372                     break
3373         if bad_changesfile:
3374             die("Option --changesfile is incompatible with revision specifiers")
3375
3376         newPaths = []
3377         for p in self.depotPaths:
3378             if p.find("@") != -1:
3379                 atIdx = p.index("@")
3380                 self.changeRange = p[atIdx:]
3381                 if self.changeRange == "@all":
3382                     self.changeRange = ""
3383                 elif ',' not in self.changeRange:
3384                     revision = self.changeRange
3385                     self.changeRange = ""
3386                 p = p[:atIdx]
3387             elif p.find("#") != -1:
3388                 hashIdx = p.index("#")
3389                 revision = p[hashIdx:]
3390                 p = p[:hashIdx]
3391             elif self.previousDepotPaths == []:
3392                 # pay attention to changesfile, if given, else import
3393                 # the entire p4 tree at the head revision
3394                 if len(self.changesFile) == 0:
3395                     revision = "#head"
3396
3397             p = re.sub ("\.\.\.$", "", p)
3398             if not p.endswith("/"):
3399                 p += "/"
3400
3401             newPaths.append(p)
3402
3403         self.depotPaths = newPaths
3404
3405         # --detect-branches may change this for each branch
3406         self.branchPrefixes = self.depotPaths
3407
3408         self.loadUserMapFromCache()
3409         self.labels = {}
3410         if self.detectLabels:
3411             self.getLabels();
3412
3413         if self.detectBranches:
3414             ## FIXME - what's a P4 projectName ?
3415             self.projectName = self.guessProjectName()
3416
3417             if self.hasOrigin:
3418                 self.getBranchMappingFromGitBranches()
3419             else:
3420                 self.getBranchMapping()
3421             if self.verbose:
3422                 print "p4-git branches: %s" % self.p4BranchesInGit
3423                 print "initial parents: %s" % self.initialParents
3424             for b in self.p4BranchesInGit:
3425                 if b != "master":
3426
3427                     ## FIXME
3428                     b = b[len(self.projectName):]
3429                 self.createdBranches.add(b)
3430
3431         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3432
3433         self.importProcess = subprocess.Popen(["git", "fast-import"],
3434                                               stdin=subprocess.PIPE,
3435                                               stdout=subprocess.PIPE,
3436                                               stderr=subprocess.PIPE);
3437         self.gitOutput = self.importProcess.stdout
3438         self.gitStream = self.importProcess.stdin
3439         self.gitError = self.importProcess.stderr
3440
3441         if revision:
3442             self.importHeadRevision(revision)
3443         else:
3444             changes = []
3445
3446             if len(self.changesFile) > 0:
3447                 output = open(self.changesFile).readlines()
3448                 changeSet = set()
3449                 for line in output:
3450                     changeSet.add(int(line))
3451
3452                 for change in changeSet:
3453                     changes.append(change)
3454
3455                 changes.sort()
3456             else:
3457                 # catch "git p4 sync" with no new branches, in a repo that
3458                 # does not have any existing p4 branches
3459                 if len(args) == 0:
3460                     if not self.p4BranchesInGit:
3461                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3462
3463                     # The default branch is master, unless --branch is used to
3464                     # specify something else.  Make sure it exists, or complain
3465                     # nicely about how to use --branch.
3466                     if not self.detectBranches:
3467                         if not branch_exists(self.branch):
3468                             if branch_arg_given:
3469                                 die("Error: branch %s does not exist." % self.branch)
3470                             else:
3471                                 die("Error: no branch %s; perhaps specify one with --branch." %
3472                                     self.branch)
3473
3474                 if self.verbose:
3475                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3476                                                               self.changeRange)
3477                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3478
3479                 if len(self.maxChanges) > 0:
3480                     changes = changes[:min(int(self.maxChanges), len(changes))]
3481
3482             if len(changes) == 0:
3483                 if not self.silent:
3484                     print "No changes to import!"
3485             else:
3486                 if not self.silent and not self.detectBranches:
3487                     print "Import destination: %s" % self.branch
3488
3489                 self.updatedBranches = set()
3490
3491                 if not self.detectBranches:
3492                     if args:
3493                         # start a new branch
3494                         self.initialParent = ""
3495                     else:
3496                         # build on a previous revision
3497                         self.initialParent = parseRevision(self.branch)
3498
3499                 self.importChanges(changes)
3500
3501                 if not self.silent:
3502                     print ""
3503                     if len(self.updatedBranches) > 0:
3504                         sys.stdout.write("Updated branches: ")
3505                         for b in self.updatedBranches:
3506                             sys.stdout.write("%s " % b)
3507                         sys.stdout.write("\n")
3508
3509         if gitConfigBool("git-p4.importLabels"):
3510             self.importLabels = True
3511
3512         if self.importLabels:
3513             p4Labels = getP4Labels(self.depotPaths)
3514             gitTags = getGitTags()
3515
3516             missingP4Labels = p4Labels - gitTags
3517             self.importP4Labels(self.gitStream, missingP4Labels)
3518
3519         self.gitStream.close()
3520         if self.importProcess.wait() != 0:
3521             die("fast-import failed: %s" % self.gitError.read())
3522         self.gitOutput.close()
3523         self.gitError.close()
3524
3525         # Cleanup temporary branches created during import
3526         if self.tempBranches != []:
3527             for branch in self.tempBranches:
3528                 read_pipe("git update-ref -d %s" % branch)
3529             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3530
3531         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3532         # a convenient shortcut refname "p4".
3533         if self.importIntoRemotes:
3534             head_ref = self.refPrefix + "HEAD"
3535             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3536                 system(["git", "symbolic-ref", head_ref, self.branch])
3537
3538         return True
3539
3540 class P4Rebase(Command):
3541     def __init__(self):
3542         Command.__init__(self)
3543         self.options = [
3544                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3545         ]
3546         self.importLabels = False
3547         self.description = ("Fetches the latest revision from perforce and "
3548                             + "rebases the current work (branch) against it")
3549
3550     def run(self, args):
3551         sync = P4Sync()
3552         sync.importLabels = self.importLabels
3553         sync.run([])
3554
3555         return self.rebase()
3556
3557     def rebase(self):
3558         if os.system("git update-index --refresh") != 0:
3559             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3560         if len(read_pipe("git diff-index HEAD --")) > 0:
3561             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3562
3563         [upstream, settings] = findUpstreamBranchPoint()
3564         if len(upstream) == 0:
3565             die("Cannot find upstream branchpoint for rebase")
3566
3567         # the branchpoint may be p4/foo~3, so strip off the parent
3568         upstream = re.sub("~[0-9]+$", "", upstream)
3569
3570         print "Rebasing the current branch onto %s" % upstream
3571         oldHead = read_pipe("git rev-parse HEAD").strip()
3572         system("git rebase %s" % upstream)
3573         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3574         return True
3575
3576 class P4Clone(P4Sync):
3577     def __init__(self):
3578         P4Sync.__init__(self)
3579         self.description = "Creates a new git repository and imports from Perforce into it"
3580         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3581         self.options += [
3582             optparse.make_option("--destination", dest="cloneDestination",
3583                                  action='store', default=None,
3584                                  help="where to leave result of the clone"),
3585             optparse.make_option("--bare", dest="cloneBare",
3586                                  action="store_true", default=False),
3587         ]
3588         self.cloneDestination = None
3589         self.needsGit = False
3590         self.cloneBare = False
3591
3592     def defaultDestination(self, args):
3593         ## TODO: use common prefix of args?
3594         depotPath = args[0]
3595         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3596         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3597         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3598         depotDir = re.sub(r"/$", "", depotDir)
3599         return os.path.split(depotDir)[1]
3600
3601     def run(self, args):
3602         if len(args) < 1:
3603             return False
3604
3605         if self.keepRepoPath and not self.cloneDestination:
3606             sys.stderr.write("Must specify destination for --keep-path\n")
3607             sys.exit(1)
3608
3609         depotPaths = args
3610
3611         if not self.cloneDestination and len(depotPaths) > 1:
3612             self.cloneDestination = depotPaths[-1]
3613             depotPaths = depotPaths[:-1]
3614
3615         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3616         for p in depotPaths:
3617             if not p.startswith("//"):
3618                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3619                 return False
3620
3621         if not self.cloneDestination:
3622             self.cloneDestination = self.defaultDestination(args)
3623
3624         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3625
3626         if not os.path.exists(self.cloneDestination):
3627             os.makedirs(self.cloneDestination)
3628         chdir(self.cloneDestination)
3629
3630         init_cmd = [ "git", "init" ]
3631         if self.cloneBare:
3632             init_cmd.append("--bare")
3633         retcode = subprocess.call(init_cmd)
3634         if retcode:
3635             raise CalledProcessError(retcode, init_cmd)
3636
3637         if not P4Sync.run(self, depotPaths):
3638             return False
3639
3640         # create a master branch and check out a work tree
3641         if gitBranchExists(self.branch):
3642             system([ "git", "branch", "master", self.branch ])
3643             if not self.cloneBare:
3644                 system([ "git", "checkout", "-f" ])
3645         else:
3646             print 'Not checking out any branch, use ' \
3647                   '"git checkout -q -b master <branch>"'
3648
3649         # auto-set this variable if invoked with --use-client-spec
3650         if self.useClientSpec_from_options:
3651             system("git config --bool git-p4.useclientspec true")
3652
3653         return True
3654
3655 class P4Branches(Command):
3656     def __init__(self):
3657         Command.__init__(self)
3658         self.options = [ ]
3659         self.description = ("Shows the git branches that hold imports and their "
3660                             + "corresponding perforce depot paths")
3661         self.verbose = False
3662
3663     def run(self, args):
3664         if originP4BranchesExist():
3665             createOrUpdateBranchesFromOrigin()
3666
3667         cmdline = "git rev-parse --symbolic "
3668         cmdline += " --remotes"
3669
3670         for line in read_pipe_lines(cmdline):
3671             line = line.strip()
3672
3673             if not line.startswith('p4/') or line == "p4/HEAD":
3674                 continue
3675             branch = line
3676
3677             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3678             settings = extractSettingsGitLog(log)
3679
3680             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3681         return True
3682
3683 class HelpFormatter(optparse.IndentedHelpFormatter):
3684     def __init__(self):
3685         optparse.IndentedHelpFormatter.__init__(self)
3686
3687     def format_description(self, description):
3688         if description:
3689             return description + "\n"
3690         else:
3691             return ""
3692
3693 def printUsage(commands):
3694     print "usage: %s <command> [options]" % sys.argv[0]
3695     print ""
3696     print "valid commands: %s" % ", ".join(commands)
3697     print ""
3698     print "Try %s <command> --help for command specific help." % sys.argv[0]
3699     print ""
3700
3701 commands = {
3702     "debug" : P4Debug,
3703     "submit" : P4Submit,
3704     "commit" : P4Submit,
3705     "sync" : P4Sync,
3706     "rebase" : P4Rebase,
3707     "clone" : P4Clone,
3708     "rollback" : P4RollBack,
3709     "branches" : P4Branches
3710 }
3711
3712
3713 def main():
3714     if len(sys.argv[1:]) == 0:
3715         printUsage(commands.keys())
3716         sys.exit(2)
3717
3718     cmdName = sys.argv[1]
3719     try:
3720         klass = commands[cmdName]
3721         cmd = klass()
3722     except KeyError:
3723         print "unknown command %s" % cmdName
3724         print ""
3725         printUsage(commands.keys())
3726         sys.exit(2)
3727
3728     options = cmd.options
3729     cmd.gitdir = os.environ.get("GIT_DIR", None)
3730
3731     args = sys.argv[2:]
3732
3733     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3734     if cmd.needsGit:
3735         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3736
3737     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3738                                    options,
3739                                    description = cmd.description,
3740                                    formatter = HelpFormatter())
3741
3742     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3743     global verbose
3744     verbose = cmd.verbose
3745     if cmd.needsGit:
3746         if cmd.gitdir == None:
3747             cmd.gitdir = os.path.abspath(".git")
3748             if not isValidGitDir(cmd.gitdir):
3749                 # "rev-parse --git-dir" without arguments will try $PWD/.git
3750                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3751                 if os.path.exists(cmd.gitdir):
3752                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3753                     if len(cdup) > 0:
3754                         chdir(cdup);
3755
3756         if not isValidGitDir(cmd.gitdir):
3757             if isValidGitDir(cmd.gitdir + "/.git"):
3758                 cmd.gitdir += "/.git"
3759             else:
3760                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3761
3762         # so git commands invoked from the P4 workspace will succeed
3763         os.environ["GIT_DIR"] = cmd.gitdir
3764
3765     if not cmd.run(args):
3766         parser.print_help()
3767         sys.exit(2)
3768
3769
3770 if __name__ == '__main__':
3771     main()