git-p4: allow submit to create shelved changelists.
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28
29 try:
30     from subprocess import CalledProcessError
31 except ImportError:
32     # from python2.7:subprocess.py
33     # Exception classes used by this module.
34     class CalledProcessError(Exception):
35         """This exception is raised when a process run by check_call() returns
36         a non-zero exit status.  The exit status will be stored in the
37         returncode attribute."""
38         def __init__(self, returncode, cmd):
39             self.returncode = returncode
40             self.cmd = cmd
41         def __str__(self):
42             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
44 verbose = False
45
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
48
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
51
52 def p4_build_cmd(cmd):
53     """Build a suitable p4 command line.
54
55     This consolidates building and returning a p4 command line into one
56     location. It means that hooking into the environment, or other configuration
57     can be done more easily.
58     """
59     real_cmd = ["p4"]
60
61     user = gitConfig("git-p4.user")
62     if len(user) > 0:
63         real_cmd += ["-u",user]
64
65     password = gitConfig("git-p4.password")
66     if len(password) > 0:
67         real_cmd += ["-P", password]
68
69     port = gitConfig("git-p4.port")
70     if len(port) > 0:
71         real_cmd += ["-p", port]
72
73     host = gitConfig("git-p4.host")
74     if len(host) > 0:
75         real_cmd += ["-H", host]
76
77     client = gitConfig("git-p4.client")
78     if len(client) > 0:
79         real_cmd += ["-c", client]
80
81
82     if isinstance(cmd,basestring):
83         real_cmd = ' '.join(real_cmd) + ' ' + cmd
84     else:
85         real_cmd += cmd
86     return real_cmd
87
88 def chdir(path, is_client_path=False):
89     """Do chdir to the given path, and set the PWD environment
90        variable for use by P4.  It does not look at getcwd() output.
91        Since we're not using the shell, it is necessary to set the
92        PWD environment variable explicitly.
93
94        Normally, expand the path to force it to be absolute.  This
95        addresses the use of relative path names inside P4 settings,
96        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
97        as given; it looks for .p4config using PWD.
98
99        If is_client_path, the path was handed to us directly by p4,
100        and may be a symbolic link.  Do not call os.getcwd() in this
101        case, because it will cause p4 to think that PWD is not inside
102        the client path.
103        """
104
105     os.chdir(path)
106     if not is_client_path:
107         path = os.getcwd()
108     os.environ['PWD'] = path
109
110 def calcDiskFree():
111     """Return free space in bytes on the disk of the given dirname."""
112     if platform.system() == 'Windows':
113         free_bytes = ctypes.c_ulonglong(0)
114         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115         return free_bytes.value
116     else:
117         st = os.statvfs(os.getcwd())
118         return st.f_bavail * st.f_frsize
119
120 def die(msg):
121     if verbose:
122         raise Exception(msg)
123     else:
124         sys.stderr.write(msg + "\n")
125         sys.exit(1)
126
127 def write_pipe(c, stdin):
128     if verbose:
129         sys.stderr.write('Writing pipe: %s\n' % str(c))
130
131     expand = isinstance(c,basestring)
132     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133     pipe = p.stdin
134     val = pipe.write(stdin)
135     pipe.close()
136     if p.wait():
137         die('Command failed: %s' % str(c))
138
139     return val
140
141 def p4_write_pipe(c, stdin):
142     real_cmd = p4_build_cmd(c)
143     return write_pipe(real_cmd, stdin)
144
145 def read_pipe(c, ignore_error=False):
146     if verbose:
147         sys.stderr.write('Reading pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151     (out, err) = p.communicate()
152     if p.returncode != 0 and not ignore_error:
153         die('Command failed: %s\nError: %s' % (str(c), err))
154     return out
155
156 def p4_read_pipe(c, ignore_error=False):
157     real_cmd = p4_build_cmd(c)
158     return read_pipe(real_cmd, ignore_error)
159
160 def read_pipe_lines(c):
161     if verbose:
162         sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164     expand = isinstance(c, basestring)
165     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166     pipe = p.stdout
167     val = pipe.readlines()
168     if pipe.close() or p.wait():
169         die('Command failed: %s' % str(c))
170
171     return val
172
173 def p4_read_pipe_lines(c):
174     """Specifically invoke p4 on the command supplied. """
175     real_cmd = p4_build_cmd(c)
176     return read_pipe_lines(real_cmd)
177
178 def p4_has_command(cmd):
179     """Ask p4 for help on this command.  If it returns an error, the
180        command does not exist in this version of p4."""
181     real_cmd = p4_build_cmd(["help", cmd])
182     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183                                    stderr=subprocess.PIPE)
184     p.communicate()
185     return p.returncode == 0
186
187 def p4_has_move_command():
188     """See if the move command exists, that it supports -k, and that
189        it has not been administratively disabled.  The arguments
190        must be correct, but the filenames do not have to exist.  Use
191        ones with wildcards so even if they exist, it will fail."""
192
193     if not p4_has_command("move"):
194         return False
195     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197     (out, err) = p.communicate()
198     # return code will be 1 in either case
199     if err.find("Invalid option") >= 0:
200         return False
201     if err.find("disabled") >= 0:
202         return False
203     # assume it failed because @... was invalid changelist
204     return True
205
206 def system(cmd, ignore_error=False):
207     expand = isinstance(cmd,basestring)
208     if verbose:
209         sys.stderr.write("executing %s\n" % str(cmd))
210     retcode = subprocess.call(cmd, shell=expand)
211     if retcode and not ignore_error:
212         raise CalledProcessError(retcode, cmd)
213
214     return retcode
215
216 def p4_system(cmd):
217     """Specifically invoke p4 as the system command. """
218     real_cmd = p4_build_cmd(cmd)
219     expand = isinstance(real_cmd, basestring)
220     retcode = subprocess.call(real_cmd, shell=expand)
221     if retcode:
222         raise CalledProcessError(retcode, real_cmd)
223
224 _p4_version_string = None
225 def p4_version_string():
226     """Read the version string, showing just the last line, which
227        hopefully is the interesting version bit.
228
229        $ p4 -V
230        Perforce - The Fast Software Configuration Management System.
231        Copyright 1995-2011 Perforce Software.  All rights reserved.
232        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233     """
234     global _p4_version_string
235     if not _p4_version_string:
236         a = p4_read_pipe_lines(["-V"])
237         _p4_version_string = a[-1].rstrip()
238     return _p4_version_string
239
240 def p4_integrate(src, dest):
241     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
242
243 def p4_sync(f, *options):
244     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
245
246 def p4_add(f):
247     # forcibly add file names with wildcards
248     if wildcard_present(f):
249         p4_system(["add", "-f", f])
250     else:
251         p4_system(["add", f])
252
253 def p4_delete(f):
254     p4_system(["delete", wildcard_encode(f)])
255
256 def p4_edit(f, *options):
257     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
258
259 def p4_revert(f):
260     p4_system(["revert", wildcard_encode(f)])
261
262 def p4_reopen(type, f):
263     p4_system(["reopen", "-t", type, wildcard_encode(f)])
264
265 def p4_move(src, dest):
266     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
267
268 def p4_last_change():
269     results = p4CmdList(["changes", "-m", "1"])
270     return int(results[0]['change'])
271
272 def p4_describe(change):
273     """Make sure it returns a valid result by checking for
274        the presence of field "time".  Return a dict of the
275        results."""
276
277     ds = p4CmdList(["describe", "-s", str(change)])
278     if len(ds) != 1:
279         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280
281     d = ds[0]
282
283     if "p4ExitCode" in d:
284         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
285                                                       str(d)))
286     if "code" in d:
287         if d["code"] == "error":
288             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
289
290     if "time" not in d:
291         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
292
293     return d
294
295 #
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers.  See "p4 help filetypes"
298 # for a list and explanation.
299 #
300 def split_p4_type(p4type):
301
302     p4_filetypes_historical = {
303         "ctempobj": "binary+Sw",
304         "ctext": "text+C",
305         "cxtext": "text+Cx",
306         "ktext": "text+k",
307         "kxtext": "text+kx",
308         "ltext": "text+F",
309         "tempobj": "binary+FSw",
310         "ubinary": "binary+F",
311         "uresource": "resource+F",
312         "uxbinary": "binary+Fx",
313         "xbinary": "binary+x",
314         "xltext": "text+Fx",
315         "xtempobj": "binary+Swx",
316         "xtext": "text+x",
317         "xunicode": "unicode+x",
318         "xutf16": "utf16+x",
319     }
320     if p4type in p4_filetypes_historical:
321         p4type = p4_filetypes_historical[p4type]
322     mods = ""
323     s = p4type.split("+")
324     base = s[0]
325     mods = ""
326     if len(s) > 1:
327         mods = s[1]
328     return (base, mods)
329
330 #
331 # return the raw p4 type of a file (text, text+ko, etc)
332 #
333 def p4_type(f):
334     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
335     return results[0]['headType']
336
337 #
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
340 #
341 def p4_keywords_regexp_for_type(base, type_mods):
342     if base in ("text", "unicode", "binary"):
343         kwords = None
344         if "ko" in type_mods:
345             kwords = 'Id|Header'
346         elif "k" in type_mods:
347             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348         else:
349             return None
350         pattern = r"""
351             \$              # Starts with a dollar, followed by...
352             (%s)            # one of the keywords, followed by...
353             (:[^$\n]+)?     # possibly an old expansion, followed by...
354             \$              # another dollar
355             """ % kwords
356         return pattern
357     else:
358         return None
359
360 #
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
364 #
365 def p4_keywords_regexp_for_file(file):
366     if not os.path.exists(file):
367         return None
368     else:
369         (type_base, type_mods) = split_p4_type(p4_type(file))
370         return p4_keywords_regexp_for_type(type_base, type_mods)
371
372 def setP4ExecBit(file, mode):
373     # Reopens an already open file and changes the execute bit to match
374     # the execute bit setting in the passed in mode.
375
376     p4Type = "+x"
377
378     if not isModeExec(mode):
379         p4Type = getP4OpenedType(file)
380         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382         if p4Type[-1] == "+":
383             p4Type = p4Type[0:-1]
384
385     p4_reopen(p4Type, file)
386
387 def getP4OpenedType(file):
388     # Returns the perforce file type for the given file.
389
390     result = p4_read_pipe(["opened", wildcard_encode(file)])
391     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
392     if match:
393         return match.group(1)
394     else:
395         die("Could not determine file type for %s (result: '%s')" % (file, result))
396
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths):
399     labels = set()
400     if isinstance(depotPaths,basestring):
401         depotPaths = [depotPaths]
402
403     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
404         label = l['label']
405         labels.add(label)
406
407     return labels
408
409 # Return the set of all git tags
410 def getGitTags():
411     gitTags = set()
412     for line in read_pipe_lines(["git", "tag"]):
413         tag = line.strip()
414         gitTags.add(tag)
415     return gitTags
416
417 def diffTreePattern():
418     # This is a simple generator for the diff tree regex pattern. This could be
419     # a class variable if this and parseDiffTreeEntry were a part of a class.
420     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421     while True:
422         yield pattern
423
424 def parseDiffTreeEntry(entry):
425     """Parses a single diff tree entry into its component elements.
426
427     See git-diff-tree(1) manpage for details about the format of the diff
428     output. This method returns a dictionary with the following elements:
429
430     src_mode - The mode of the source file
431     dst_mode - The mode of the destination file
432     src_sha1 - The sha1 for the source file
433     dst_sha1 - The sha1 fr the destination file
434     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435     status_score - The score for the status (applicable for 'C' and 'R'
436                    statuses). This is None if there is no score.
437     src - The path for the source file.
438     dst - The path for the destination file. This is only present for
439           copy or renames. If it is not present, this is None.
440
441     If the pattern is not matched, None is returned."""
442
443     match = diffTreePattern().next().match(entry)
444     if match:
445         return {
446             'src_mode': match.group(1),
447             'dst_mode': match.group(2),
448             'src_sha1': match.group(3),
449             'dst_sha1': match.group(4),
450             'status': match.group(5),
451             'status_score': match.group(6),
452             'src': match.group(7),
453             'dst': match.group(10)
454         }
455     return None
456
457 def isModeExec(mode):
458     # Returns True if the given git mode represents an executable file,
459     # otherwise False.
460     return mode[-3:] == "755"
461
462 def isModeExecChanged(src_mode, dst_mode):
463     return isModeExec(src_mode) != isModeExec(dst_mode)
464
465 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
466
467     if isinstance(cmd,basestring):
468         cmd = "-G " + cmd
469         expand = True
470     else:
471         cmd = ["-G"] + cmd
472         expand = False
473
474     cmd = p4_build_cmd(cmd)
475     if verbose:
476         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
477
478     # Use a temporary file to avoid deadlocks without
479     # subprocess.communicate(), which would put another copy
480     # of stdout into memory.
481     stdin_file = None
482     if stdin is not None:
483         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
484         if isinstance(stdin,basestring):
485             stdin_file.write(stdin)
486         else:
487             for i in stdin:
488                 stdin_file.write(i + '\n')
489         stdin_file.flush()
490         stdin_file.seek(0)
491
492     p4 = subprocess.Popen(cmd,
493                           shell=expand,
494                           stdin=stdin_file,
495                           stdout=subprocess.PIPE)
496
497     result = []
498     try:
499         while True:
500             entry = marshal.load(p4.stdout)
501             if cb is not None:
502                 cb(entry)
503             else:
504                 result.append(entry)
505     except EOFError:
506         pass
507     exitCode = p4.wait()
508     if exitCode != 0:
509         entry = {}
510         entry["p4ExitCode"] = exitCode
511         result.append(entry)
512
513     return result
514
515 def p4Cmd(cmd):
516     list = p4CmdList(cmd)
517     result = {}
518     for entry in list:
519         result.update(entry)
520     return result;
521
522 def p4Where(depotPath):
523     if not depotPath.endswith("/"):
524         depotPath += "/"
525     depotPathLong = depotPath + "..."
526     outputList = p4CmdList(["where", depotPathLong])
527     output = None
528     for entry in outputList:
529         if "depotFile" in entry:
530             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531             # The base path always ends with "/...".
532             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
533                 output = entry
534                 break
535         elif "data" in entry:
536             data = entry.get("data")
537             space = data.find(" ")
538             if data[:space] == depotPath:
539                 output = entry
540                 break
541     if output == None:
542         return ""
543     if output["code"] == "error":
544         return ""
545     clientPath = ""
546     if "path" in output:
547         clientPath = output.get("path")
548     elif "data" in output:
549         data = output.get("data")
550         lastSpace = data.rfind(" ")
551         clientPath = data[lastSpace + 1:]
552
553     if clientPath.endswith("..."):
554         clientPath = clientPath[:-3]
555     return clientPath
556
557 def currentGitBranch():
558     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
559     if retcode != 0:
560         # on a detached head
561         return None
562     else:
563         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
564
565 def isValidGitDir(path):
566     if (os.path.exists(path + "/HEAD")
567         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
568         return True;
569     return False
570
571 def parseRevision(ref):
572     return read_pipe("git rev-parse %s" % ref).strip()
573
574 def branchExists(ref):
575     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
576                      ignore_error=True)
577     return len(rev) > 0
578
579 def extractLogMessageFromGitCommit(commit):
580     logMessage = ""
581
582     ## fixme: title is first line of commit, not 1st paragraph.
583     foundTitle = False
584     for log in read_pipe_lines("git cat-file commit %s" % commit):
585        if not foundTitle:
586            if len(log) == 1:
587                foundTitle = True
588            continue
589
590        logMessage += log
591     return logMessage
592
593 def extractSettingsGitLog(log):
594     values = {}
595     for line in log.split("\n"):
596         line = line.strip()
597         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
598         if not m:
599             continue
600
601         assignments = m.group(1).split (':')
602         for a in assignments:
603             vals = a.split ('=')
604             key = vals[0].strip()
605             val = ('='.join (vals[1:])).strip()
606             if val.endswith ('\"') and val.startswith('"'):
607                 val = val[1:-1]
608
609             values[key] = val
610
611     paths = values.get("depot-paths")
612     if not paths:
613         paths = values.get("depot-path")
614     if paths:
615         values['depot-paths'] = paths.split(',')
616     return values
617
618 def gitBranchExists(branch):
619     proc = subprocess.Popen(["git", "rev-parse", branch],
620                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
621     return proc.wait() == 0;
622
623 _gitConfig = {}
624
625 def gitConfig(key, typeSpecifier=None):
626     if not _gitConfig.has_key(key):
627         cmd = [ "git", "config" ]
628         if typeSpecifier:
629             cmd += [ typeSpecifier ]
630         cmd += [ key ]
631         s = read_pipe(cmd, ignore_error=True)
632         _gitConfig[key] = s.strip()
633     return _gitConfig[key]
634
635 def gitConfigBool(key):
636     """Return a bool, using git config --bool.  It is True only if the
637        variable is set to true, and False if set to false or not present
638        in the config."""
639
640     if not _gitConfig.has_key(key):
641         _gitConfig[key] = gitConfig(key, '--bool') == "true"
642     return _gitConfig[key]
643
644 def gitConfigInt(key):
645     if not _gitConfig.has_key(key):
646         cmd = [ "git", "config", "--int", key ]
647         s = read_pipe(cmd, ignore_error=True)
648         v = s.strip()
649         try:
650             _gitConfig[key] = int(gitConfig(key, '--int'))
651         except ValueError:
652             _gitConfig[key] = None
653     return _gitConfig[key]
654
655 def gitConfigList(key):
656     if not _gitConfig.has_key(key):
657         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658         _gitConfig[key] = s.strip().split(os.linesep)
659         if _gitConfig[key] == ['']:
660             _gitConfig[key] = []
661     return _gitConfig[key]
662
663 def p4BranchesInGit(branchesAreInRemotes=True):
664     """Find all the branches whose names start with "p4/", looking
665        in remotes or heads as specified by the argument.  Return
666        a dictionary of { branch: revision } for each one found.
667        The branch names are the short names, without any
668        "p4/" prefix."""
669
670     branches = {}
671
672     cmdline = "git rev-parse --symbolic "
673     if branchesAreInRemotes:
674         cmdline += "--remotes"
675     else:
676         cmdline += "--branches"
677
678     for line in read_pipe_lines(cmdline):
679         line = line.strip()
680
681         # only import to p4/
682         if not line.startswith('p4/'):
683             continue
684         # special symbolic ref to p4/master
685         if line == "p4/HEAD":
686             continue
687
688         # strip off p4/ prefix
689         branch = line[len("p4/"):]
690
691         branches[branch] = parseRevision(line)
692
693     return branches
694
695 def branch_exists(branch):
696     """Make sure that the given ref name really exists."""
697
698     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700     out, _ = p.communicate()
701     if p.returncode:
702         return False
703     # expect exactly one line of output: the branch name
704     return out.rstrip() == branch
705
706 def findUpstreamBranchPoint(head = "HEAD"):
707     branches = p4BranchesInGit()
708     # map from depot-path to branch name
709     branchByDepotPath = {}
710     for branch in branches.keys():
711         tip = branches[branch]
712         log = extractLogMessageFromGitCommit(tip)
713         settings = extractSettingsGitLog(log)
714         if settings.has_key("depot-paths"):
715             paths = ",".join(settings["depot-paths"])
716             branchByDepotPath[paths] = "remotes/p4/" + branch
717
718     settings = None
719     parent = 0
720     while parent < 65535:
721         commit = head + "~%s" % parent
722         log = extractLogMessageFromGitCommit(commit)
723         settings = extractSettingsGitLog(log)
724         if settings.has_key("depot-paths"):
725             paths = ",".join(settings["depot-paths"])
726             if branchByDepotPath.has_key(paths):
727                 return [branchByDepotPath[paths], settings]
728
729         parent = parent + 1
730
731     return ["", settings]
732
733 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
734     if not silent:
735         print ("Creating/updating branch(es) in %s based on origin branch(es)"
736                % localRefPrefix)
737
738     originPrefix = "origin/p4/"
739
740     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
741         line = line.strip()
742         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
743             continue
744
745         headName = line[len(originPrefix):]
746         remoteHead = localRefPrefix + headName
747         originHead = line
748
749         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750         if (not original.has_key('depot-paths')
751             or not original.has_key('change')):
752             continue
753
754         update = False
755         if not gitBranchExists(remoteHead):
756             if verbose:
757                 print "creating %s" % remoteHead
758             update = True
759         else:
760             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761             if settings.has_key('change') > 0:
762                 if settings['depot-paths'] == original['depot-paths']:
763                     originP4Change = int(original['change'])
764                     p4Change = int(settings['change'])
765                     if originP4Change > p4Change:
766                         print ("%s (%s) is newer than %s (%s). "
767                                "Updating p4 branch from origin."
768                                % (originHead, originP4Change,
769                                   remoteHead, p4Change))
770                         update = True
771                 else:
772                     print ("Ignoring: %s was imported from %s while "
773                            "%s was imported from %s"
774                            % (originHead, ','.join(original['depot-paths']),
775                               remoteHead, ','.join(settings['depot-paths'])))
776
777         if update:
778             system("git update-ref %s %s" % (remoteHead, originHead))
779
780 def originP4BranchesExist():
781         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
782
783
784 def p4ParseNumericChangeRange(parts):
785     changeStart = int(parts[0][1:])
786     if parts[1] == '#head':
787         changeEnd = p4_last_change()
788     else:
789         changeEnd = int(parts[1])
790
791     return (changeStart, changeEnd)
792
793 def chooseBlockSize(blockSize):
794     if blockSize:
795         return blockSize
796     else:
797         return defaultBlockSize
798
799 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
800     assert depotPaths
801
802     # Parse the change range into start and end. Try to find integer
803     # revision ranges as these can be broken up into blocks to avoid
804     # hitting server-side limits (maxrows, maxscanresults). But if
805     # that doesn't work, fall back to using the raw revision specifier
806     # strings, without using block mode.
807
808     if changeRange is None or changeRange == '':
809         changeStart = 1
810         changeEnd = p4_last_change()
811         block_size = chooseBlockSize(requestedBlockSize)
812     else:
813         parts = changeRange.split(',')
814         assert len(parts) == 2
815         try:
816             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817             block_size = chooseBlockSize(requestedBlockSize)
818         except:
819             changeStart = parts[0][1:]
820             changeEnd = parts[1]
821             if requestedBlockSize:
822                 die("cannot use --changes-block-size with non-numeric revisions")
823             block_size = None
824
825     changes = []
826
827     # Retrieve changes a block at a time, to prevent running
828     # into a MaxResults/MaxScanRows error from the server.
829
830     while True:
831         cmd = ['changes']
832
833         if block_size:
834             end = min(changeEnd, changeStart + block_size)
835             revisionRange = "%d,%d" % (changeStart, end)
836         else:
837             revisionRange = "%s,%s" % (changeStart, changeEnd)
838
839         for p in depotPaths:
840             cmd += ["%s...@%s" % (p, revisionRange)]
841
842         # Insert changes in chronological order
843         for line in reversed(p4_read_pipe_lines(cmd)):
844             changes.append(int(line.split(" ")[1]))
845
846         if not block_size:
847             break
848
849         if end >= changeEnd:
850             break
851
852         changeStart = end + 1
853
854     changes = sorted(changes)
855     return changes
856
857 def p4PathStartsWith(path, prefix):
858     # This method tries to remedy a potential mixed-case issue:
859     #
860     # If UserA adds  //depot/DirA/file1
861     # and UserB adds //depot/dira/file2
862     #
863     # we may or may not have a problem. If you have core.ignorecase=true,
864     # we treat DirA and dira as the same directory
865     if gitConfigBool("core.ignorecase"):
866         return path.lower().startswith(prefix.lower())
867     return path.startswith(prefix)
868
869 def getClientSpec():
870     """Look at the p4 client spec, create a View() object that contains
871        all the mappings, and return it."""
872
873     specList = p4CmdList("client -o")
874     if len(specList) != 1:
875         die('Output from "client -o" is %d lines, expecting 1' %
876             len(specList))
877
878     # dictionary of all client parameters
879     entry = specList[0]
880
881     # the //client/ name
882     client_name = entry["Client"]
883
884     # just the keys that start with "View"
885     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
886
887     # hold this new View
888     view = View(client_name)
889
890     # append the lines, in order, to the view
891     for view_num in range(len(view_keys)):
892         k = "View%d" % view_num
893         if k not in view_keys:
894             die("Expected view key %s missing" % k)
895         view.append(entry[k])
896
897     return view
898
899 def getClientRoot():
900     """Grab the client directory."""
901
902     output = p4CmdList("client -o")
903     if len(output) != 1:
904         die('Output from "client -o" is %d lines, expecting 1' % len(output))
905
906     entry = output[0]
907     if "Root" not in entry:
908         die('Client has no "Root"')
909
910     return entry["Root"]
911
912 #
913 # P4 wildcards are not allowed in filenames.  P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
916 #
917 def wildcard_decode(path):
918     # Search for and fix just these four characters.  Do % last so
919     # that fixing it does not inadvertently create new %-escapes.
920     # Cannot have * in a filename in windows; untested as to
921     # what p4 would do in such a case.
922     if not platform.system() == "Windows":
923         path = path.replace("%2A", "*")
924     path = path.replace("%23", "#") \
925                .replace("%40", "@") \
926                .replace("%25", "%")
927     return path
928
929 def wildcard_encode(path):
930     # do % first to avoid double-encoding the %s introduced here
931     path = path.replace("%", "%25") \
932                .replace("*", "%2A") \
933                .replace("#", "%23") \
934                .replace("@", "%40")
935     return path
936
937 def wildcard_present(path):
938     m = re.search("[*#@%]", path)
939     return m is not None
940
941 class LargeFileSystem(object):
942     """Base class for large file system support."""
943
944     def __init__(self, writeToGitStream):
945         self.largeFiles = set()
946         self.writeToGitStream = writeToGitStream
947
948     def generatePointer(self, cloneDestination, contentFile):
949         """Return the content of a pointer file that is stored in Git instead of
950            the actual content."""
951         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
952
953     def pushFile(self, localLargeFile):
954         """Push the actual content which is not stored in the Git repository to
955            a server."""
956         assert False, "Method 'pushFile' required in " + self.__class__.__name__
957
958     def hasLargeFileExtension(self, relPath):
959         return reduce(
960             lambda a, b: a or b,
961             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
962             False
963         )
964
965     def generateTempFile(self, contents):
966         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
967         for d in contents:
968             contentFile.write(d)
969         contentFile.close()
970         return contentFile.name
971
972     def exceedsLargeFileThreshold(self, relPath, contents):
973         if gitConfigInt('git-p4.largeFileThreshold'):
974             contentsSize = sum(len(d) for d in contents)
975             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
976                 return True
977         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978             contentsSize = sum(len(d) for d in contents)
979             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
980                 return False
981             contentTempFile = self.generateTempFile(contents)
982             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
985             zf.close()
986             compressedContentsSize = zf.infolist()[0].compress_size
987             os.remove(contentTempFile)
988             os.remove(compressedContentFile.name)
989             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
990                 return True
991         return False
992
993     def addLargeFile(self, relPath):
994         self.largeFiles.add(relPath)
995
996     def removeLargeFile(self, relPath):
997         self.largeFiles.remove(relPath)
998
999     def isLargeFile(self, relPath):
1000         return relPath in self.largeFiles
1001
1002     def processContent(self, git_mode, relPath, contents):
1003         """Processes the content of git fast import. This method decides if a
1004            file is stored in the large file system and handles all necessary
1005            steps."""
1006         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007             contentTempFile = self.generateTempFile(contents)
1008             (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1009
1010             # Move temp file to final location in large file system
1011             largeFileDir = os.path.dirname(localLargeFile)
1012             if not os.path.isdir(largeFileDir):
1013                 os.makedirs(largeFileDir)
1014             shutil.move(contentTempFile, localLargeFile)
1015             self.addLargeFile(relPath)
1016             if gitConfigBool('git-p4.largeFilePush'):
1017                 self.pushFile(localLargeFile)
1018             if verbose:
1019                 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020         return (git_mode, contents)
1021
1022 class MockLFS(LargeFileSystem):
1023     """Mock large file system for testing."""
1024
1025     def generatePointer(self, contentFile):
1026         """The pointer content is the original content prefixed with "pointer-".
1027            The local filename of the large file storage is derived from the file content.
1028            """
1029         with open(contentFile, 'r') as f:
1030             content = next(f)
1031             gitMode = '100644'
1032             pointerContents = 'pointer-' + content
1033             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034             return (gitMode, pointerContents, localLargeFile)
1035
1036     def pushFile(self, localLargeFile):
1037         """The remote filename of the large file storage is the same as the local
1038            one but in a different directory.
1039            """
1040         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041         if not os.path.exists(remotePath):
1042             os.makedirs(remotePath)
1043         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1044
1045 class GitLFS(LargeFileSystem):
1046     """Git LFS as backend for the git-p4 large file system.
1047        See https://git-lfs.github.com/ for details."""
1048
1049     def __init__(self, *args):
1050         LargeFileSystem.__init__(self, *args)
1051         self.baseGitAttributes = []
1052
1053     def generatePointer(self, contentFile):
1054         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055            mode and content which is stored in the Git repository instead of
1056            the actual content. Return also the new location of the actual
1057            content.
1058            """
1059         pointerProcess = subprocess.Popen(
1060             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061             stdout=subprocess.PIPE
1062         )
1063         pointerFile = pointerProcess.stdout.read()
1064         if pointerProcess.wait():
1065             os.remove(contentFile)
1066             die('git-lfs pointer command failed. Did you install the extension?')
1067
1068         # Git LFS removed the preamble in the output of the 'pointer' command
1069         # starting from version 1.2.0. Check for the preamble here to support
1070         # earlier versions.
1071         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1072         if pointerFile.startswith('Git LFS pointer for'):
1073             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1074
1075         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1076         localLargeFile = os.path.join(
1077             os.getcwd(),
1078             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1079             oid,
1080         )
1081         # LFS Spec states that pointer files should not have the executable bit set.
1082         gitMode = '100644'
1083         return (gitMode, pointerFile, localLargeFile)
1084
1085     def pushFile(self, localLargeFile):
1086         uploadProcess = subprocess.Popen(
1087             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1088         )
1089         if uploadProcess.wait():
1090             die('git-lfs push command failed. Did you define a remote?')
1091
1092     def generateGitAttributes(self):
1093         return (
1094             self.baseGitAttributes +
1095             [
1096                 '\n',
1097                 '#\n',
1098                 '# Git LFS (see https://git-lfs.github.com/)\n',
1099                 '#\n',
1100             ] +
1101             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1102                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1103             ] +
1104             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1105                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1106             ]
1107         )
1108
1109     def addLargeFile(self, relPath):
1110         LargeFileSystem.addLargeFile(self, relPath)
1111         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1112
1113     def removeLargeFile(self, relPath):
1114         LargeFileSystem.removeLargeFile(self, relPath)
1115         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1116
1117     def processContent(self, git_mode, relPath, contents):
1118         if relPath == '.gitattributes':
1119             self.baseGitAttributes = contents
1120             return (git_mode, self.generateGitAttributes())
1121         else:
1122             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1123
1124 class Command:
1125     def __init__(self):
1126         self.usage = "usage: %prog [options]"
1127         self.needsGit = True
1128         self.verbose = False
1129
1130 class P4UserMap:
1131     def __init__(self):
1132         self.userMapFromPerforceServer = False
1133         self.myP4UserId = None
1134
1135     def p4UserId(self):
1136         if self.myP4UserId:
1137             return self.myP4UserId
1138
1139         results = p4CmdList("user -o")
1140         for r in results:
1141             if r.has_key('User'):
1142                 self.myP4UserId = r['User']
1143                 return r['User']
1144         die("Could not find your p4 user id")
1145
1146     def p4UserIsMe(self, p4User):
1147         # return True if the given p4 user is actually me
1148         me = self.p4UserId()
1149         if not p4User or p4User != me:
1150             return False
1151         else:
1152             return True
1153
1154     def getUserCacheFilename(self):
1155         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1156         return home + "/.gitp4-usercache.txt"
1157
1158     def getUserMapFromPerforceServer(self):
1159         if self.userMapFromPerforceServer:
1160             return
1161         self.users = {}
1162         self.emails = {}
1163
1164         for output in p4CmdList("users"):
1165             if not output.has_key("User"):
1166                 continue
1167             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1168             self.emails[output["Email"]] = output["User"]
1169
1170         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1171         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1172             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1173             if mapUser and len(mapUser[0]) == 3:
1174                 user = mapUser[0][0]
1175                 fullname = mapUser[0][1]
1176                 email = mapUser[0][2]
1177                 self.users[user] = fullname + " <" + email + ">"
1178                 self.emails[email] = user
1179
1180         s = ''
1181         for (key, val) in self.users.items():
1182             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1183
1184         open(self.getUserCacheFilename(), "wb").write(s)
1185         self.userMapFromPerforceServer = True
1186
1187     def loadUserMapFromCache(self):
1188         self.users = {}
1189         self.userMapFromPerforceServer = False
1190         try:
1191             cache = open(self.getUserCacheFilename(), "rb")
1192             lines = cache.readlines()
1193             cache.close()
1194             for line in lines:
1195                 entry = line.strip().split("\t")
1196                 self.users[entry[0]] = entry[1]
1197         except IOError:
1198             self.getUserMapFromPerforceServer()
1199
1200 class P4Debug(Command):
1201     def __init__(self):
1202         Command.__init__(self)
1203         self.options = []
1204         self.description = "A tool to debug the output of p4 -G."
1205         self.needsGit = False
1206
1207     def run(self, args):
1208         j = 0
1209         for output in p4CmdList(args):
1210             print 'Element: %d' % j
1211             j += 1
1212             print output
1213         return True
1214
1215 class P4RollBack(Command):
1216     def __init__(self):
1217         Command.__init__(self)
1218         self.options = [
1219             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1220         ]
1221         self.description = "A tool to debug the multi-branch import. Don't use :)"
1222         self.rollbackLocalBranches = False
1223
1224     def run(self, args):
1225         if len(args) != 1:
1226             return False
1227         maxChange = int(args[0])
1228
1229         if "p4ExitCode" in p4Cmd("changes -m 1"):
1230             die("Problems executing p4");
1231
1232         if self.rollbackLocalBranches:
1233             refPrefix = "refs/heads/"
1234             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1235         else:
1236             refPrefix = "refs/remotes/"
1237             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1238
1239         for line in lines:
1240             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1241                 line = line.strip()
1242                 ref = refPrefix + line
1243                 log = extractLogMessageFromGitCommit(ref)
1244                 settings = extractSettingsGitLog(log)
1245
1246                 depotPaths = settings['depot-paths']
1247                 change = settings['change']
1248
1249                 changed = False
1250
1251                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1252                                                            for p in depotPaths]))) == 0:
1253                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1254                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1255                     continue
1256
1257                 while change and int(change) > maxChange:
1258                     changed = True
1259                     if self.verbose:
1260                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1261                     system("git update-ref %s \"%s^\"" % (ref, ref))
1262                     log = extractLogMessageFromGitCommit(ref)
1263                     settings =  extractSettingsGitLog(log)
1264
1265
1266                     depotPaths = settings['depot-paths']
1267                     change = settings['change']
1268
1269                 if changed:
1270                     print "%s rewound to %s" % (ref, change)
1271
1272         return True
1273
1274 class P4Submit(Command, P4UserMap):
1275
1276     conflict_behavior_choices = ("ask", "skip", "quit")
1277
1278     def __init__(self):
1279         Command.__init__(self)
1280         P4UserMap.__init__(self)
1281         self.options = [
1282                 optparse.make_option("--origin", dest="origin"),
1283                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1284                 # preserve the user, requires relevant p4 permissions
1285                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1286                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1287                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1288                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1289                 optparse.make_option("--conflict", dest="conflict_behavior",
1290                                      choices=self.conflict_behavior_choices),
1291                 optparse.make_option("--branch", dest="branch"),
1292                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1293                                      help="Shelve instead of submit. Shelved files are reverted, "
1294                                      "restoring the workspace to the state before the shelve"),
1295         ]
1296         self.description = "Submit changes from git to the perforce depot."
1297         self.usage += " [name of git branch to submit into perforce depot]"
1298         self.origin = ""
1299         self.detectRenames = False
1300         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1301         self.dry_run = False
1302         self.shelve = False
1303         self.prepare_p4_only = False
1304         self.conflict_behavior = None
1305         self.isWindows = (platform.system() == "Windows")
1306         self.exportLabels = False
1307         self.p4HasMoveCommand = p4_has_move_command()
1308         self.branch = None
1309
1310         if gitConfig('git-p4.largeFileSystem'):
1311             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1312
1313     def check(self):
1314         if len(p4CmdList("opened ...")) > 0:
1315             die("You have files opened with perforce! Close them before starting the sync.")
1316
1317     def separate_jobs_from_description(self, message):
1318         """Extract and return a possible Jobs field in the commit
1319            message.  It goes into a separate section in the p4 change
1320            specification.
1321
1322            A jobs line starts with "Jobs:" and looks like a new field
1323            in a form.  Values are white-space separated on the same
1324            line or on following lines that start with a tab.
1325
1326            This does not parse and extract the full git commit message
1327            like a p4 form.  It just sees the Jobs: line as a marker
1328            to pass everything from then on directly into the p4 form,
1329            but outside the description section.
1330
1331            Return a tuple (stripped log message, jobs string)."""
1332
1333         m = re.search(r'^Jobs:', message, re.MULTILINE)
1334         if m is None:
1335             return (message, None)
1336
1337         jobtext = message[m.start():]
1338         stripped_message = message[:m.start()].rstrip()
1339         return (stripped_message, jobtext)
1340
1341     def prepareLogMessage(self, template, message, jobs):
1342         """Edits the template returned from "p4 change -o" to insert
1343            the message in the Description field, and the jobs text in
1344            the Jobs field."""
1345         result = ""
1346
1347         inDescriptionSection = False
1348
1349         for line in template.split("\n"):
1350             if line.startswith("#"):
1351                 result += line + "\n"
1352                 continue
1353
1354             if inDescriptionSection:
1355                 if line.startswith("Files:") or line.startswith("Jobs:"):
1356                     inDescriptionSection = False
1357                     # insert Jobs section
1358                     if jobs:
1359                         result += jobs + "\n"
1360                 else:
1361                     continue
1362             else:
1363                 if line.startswith("Description:"):
1364                     inDescriptionSection = True
1365                     line += "\n"
1366                     for messageLine in message.split("\n"):
1367                         line += "\t" + messageLine + "\n"
1368
1369             result += line + "\n"
1370
1371         return result
1372
1373     def patchRCSKeywords(self, file, pattern):
1374         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1375         (handle, outFileName) = tempfile.mkstemp(dir='.')
1376         try:
1377             outFile = os.fdopen(handle, "w+")
1378             inFile = open(file, "r")
1379             regexp = re.compile(pattern, re.VERBOSE)
1380             for line in inFile.readlines():
1381                 line = regexp.sub(r'$\1$', line)
1382                 outFile.write(line)
1383             inFile.close()
1384             outFile.close()
1385             # Forcibly overwrite the original file
1386             os.unlink(file)
1387             shutil.move(outFileName, file)
1388         except:
1389             # cleanup our temporary file
1390             os.unlink(outFileName)
1391             print "Failed to strip RCS keywords in %s" % file
1392             raise
1393
1394         print "Patched up RCS keywords in %s" % file
1395
1396     def p4UserForCommit(self,id):
1397         # Return the tuple (perforce user,git email) for a given git commit id
1398         self.getUserMapFromPerforceServer()
1399         gitEmail = read_pipe(["git", "log", "--max-count=1",
1400                               "--format=%ae", id])
1401         gitEmail = gitEmail.strip()
1402         if not self.emails.has_key(gitEmail):
1403             return (None,gitEmail)
1404         else:
1405             return (self.emails[gitEmail],gitEmail)
1406
1407     def checkValidP4Users(self,commits):
1408         # check if any git authors cannot be mapped to p4 users
1409         for id in commits:
1410             (user,email) = self.p4UserForCommit(id)
1411             if not user:
1412                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1413                 if gitConfigBool("git-p4.allowMissingP4Users"):
1414                     print "%s" % msg
1415                 else:
1416                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1417
1418     def lastP4Changelist(self):
1419         # Get back the last changelist number submitted in this client spec. This
1420         # then gets used to patch up the username in the change. If the same
1421         # client spec is being used by multiple processes then this might go
1422         # wrong.
1423         results = p4CmdList("client -o")        # find the current client
1424         client = None
1425         for r in results:
1426             if r.has_key('Client'):
1427                 client = r['Client']
1428                 break
1429         if not client:
1430             die("could not get client spec")
1431         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1432         for r in results:
1433             if r.has_key('change'):
1434                 return r['change']
1435         die("Could not get changelist number for last submit - cannot patch up user details")
1436
1437     def modifyChangelistUser(self, changelist, newUser):
1438         # fixup the user field of a changelist after it has been submitted.
1439         changes = p4CmdList("change -o %s" % changelist)
1440         if len(changes) != 1:
1441             die("Bad output from p4 change modifying %s to user %s" %
1442                 (changelist, newUser))
1443
1444         c = changes[0]
1445         if c['User'] == newUser: return   # nothing to do
1446         c['User'] = newUser
1447         input = marshal.dumps(c)
1448
1449         result = p4CmdList("change -f -i", stdin=input)
1450         for r in result:
1451             if r.has_key('code'):
1452                 if r['code'] == 'error':
1453                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1454             if r.has_key('data'):
1455                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1456                 return
1457         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1458
1459     def canChangeChangelists(self):
1460         # check to see if we have p4 admin or super-user permissions, either of
1461         # which are required to modify changelists.
1462         results = p4CmdList(["protects", self.depotPath])
1463         for r in results:
1464             if r.has_key('perm'):
1465                 if r['perm'] == 'admin':
1466                     return 1
1467                 if r['perm'] == 'super':
1468                     return 1
1469         return 0
1470
1471     def prepareSubmitTemplate(self):
1472         """Run "p4 change -o" to grab a change specification template.
1473            This does not use "p4 -G", as it is nice to keep the submission
1474            template in original order, since a human might edit it.
1475
1476            Remove lines in the Files section that show changes to files
1477            outside the depot path we're committing into."""
1478
1479         [upstream, settings] = findUpstreamBranchPoint()
1480
1481         template = ""
1482         inFilesSection = False
1483         for line in p4_read_pipe_lines(['change', '-o']):
1484             if line.endswith("\r\n"):
1485                 line = line[:-2] + "\n"
1486             if inFilesSection:
1487                 if line.startswith("\t"):
1488                     # path starts and ends with a tab
1489                     path = line[1:]
1490                     lastTab = path.rfind("\t")
1491                     if lastTab != -1:
1492                         path = path[:lastTab]
1493                         if settings.has_key('depot-paths'):
1494                             if not [p for p in settings['depot-paths']
1495                                     if p4PathStartsWith(path, p)]:
1496                                 continue
1497                         else:
1498                             if not p4PathStartsWith(path, self.depotPath):
1499                                 continue
1500                 else:
1501                     inFilesSection = False
1502             else:
1503                 if line.startswith("Files:"):
1504                     inFilesSection = True
1505
1506             template += line
1507
1508         return template
1509
1510     def edit_template(self, template_file):
1511         """Invoke the editor to let the user change the submission
1512            message.  Return true if okay to continue with the submit."""
1513
1514         # if configured to skip the editing part, just submit
1515         if gitConfigBool("git-p4.skipSubmitEdit"):
1516             return True
1517
1518         # look at the modification time, to check later if the user saved
1519         # the file
1520         mtime = os.stat(template_file).st_mtime
1521
1522         # invoke the editor
1523         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1524             editor = os.environ.get("P4EDITOR")
1525         else:
1526             editor = read_pipe("git var GIT_EDITOR").strip()
1527         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1528
1529         # If the file was not saved, prompt to see if this patch should
1530         # be skipped.  But skip this verification step if configured so.
1531         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1532             return True
1533
1534         # modification time updated means user saved the file
1535         if os.stat(template_file).st_mtime > mtime:
1536             return True
1537
1538         while True:
1539             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1540             if response == 'y':
1541                 return True
1542             if response == 'n':
1543                 return False
1544
1545     def get_diff_description(self, editedFiles, filesToAdd):
1546         # diff
1547         if os.environ.has_key("P4DIFF"):
1548             del(os.environ["P4DIFF"])
1549         diff = ""
1550         for editedFile in editedFiles:
1551             diff += p4_read_pipe(['diff', '-du',
1552                                   wildcard_encode(editedFile)])
1553
1554         # new file diff
1555         newdiff = ""
1556         for newFile in filesToAdd:
1557             newdiff += "==== new file ====\n"
1558             newdiff += "--- /dev/null\n"
1559             newdiff += "+++ %s\n" % newFile
1560             f = open(newFile, "r")
1561             for line in f.readlines():
1562                 newdiff += "+" + line
1563             f.close()
1564
1565         return (diff + newdiff).replace('\r\n', '\n')
1566
1567     def applyCommit(self, id):
1568         """Apply one commit, return True if it succeeded."""
1569
1570         print "Applying", read_pipe(["git", "show", "-s",
1571                                      "--format=format:%h %s", id])
1572
1573         (p4User, gitEmail) = self.p4UserForCommit(id)
1574
1575         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1576         filesToAdd = set()
1577         filesToChangeType = set()
1578         filesToDelete = set()
1579         editedFiles = set()
1580         pureRenameCopy = set()
1581         filesToChangeExecBit = {}
1582
1583         for line in diff:
1584             diff = parseDiffTreeEntry(line)
1585             modifier = diff['status']
1586             path = diff['src']
1587             if modifier == "M":
1588                 p4_edit(path)
1589                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1590                     filesToChangeExecBit[path] = diff['dst_mode']
1591                 editedFiles.add(path)
1592             elif modifier == "A":
1593                 filesToAdd.add(path)
1594                 filesToChangeExecBit[path] = diff['dst_mode']
1595                 if path in filesToDelete:
1596                     filesToDelete.remove(path)
1597             elif modifier == "D":
1598                 filesToDelete.add(path)
1599                 if path in filesToAdd:
1600                     filesToAdd.remove(path)
1601             elif modifier == "C":
1602                 src, dest = diff['src'], diff['dst']
1603                 p4_integrate(src, dest)
1604                 pureRenameCopy.add(dest)
1605                 if diff['src_sha1'] != diff['dst_sha1']:
1606                     p4_edit(dest)
1607                     pureRenameCopy.discard(dest)
1608                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1609                     p4_edit(dest)
1610                     pureRenameCopy.discard(dest)
1611                     filesToChangeExecBit[dest] = diff['dst_mode']
1612                 if self.isWindows:
1613                     # turn off read-only attribute
1614                     os.chmod(dest, stat.S_IWRITE)
1615                 os.unlink(dest)
1616                 editedFiles.add(dest)
1617             elif modifier == "R":
1618                 src, dest = diff['src'], diff['dst']
1619                 if self.p4HasMoveCommand:
1620                     p4_edit(src)        # src must be open before move
1621                     p4_move(src, dest)  # opens for (move/delete, move/add)
1622                 else:
1623                     p4_integrate(src, dest)
1624                     if diff['src_sha1'] != diff['dst_sha1']:
1625                         p4_edit(dest)
1626                     else:
1627                         pureRenameCopy.add(dest)
1628                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1629                     if not self.p4HasMoveCommand:
1630                         p4_edit(dest)   # with move: already open, writable
1631                     filesToChangeExecBit[dest] = diff['dst_mode']
1632                 if not self.p4HasMoveCommand:
1633                     if self.isWindows:
1634                         os.chmod(dest, stat.S_IWRITE)
1635                     os.unlink(dest)
1636                     filesToDelete.add(src)
1637                 editedFiles.add(dest)
1638             elif modifier == "T":
1639                 filesToChangeType.add(path)
1640             else:
1641                 die("unknown modifier %s for %s" % (modifier, path))
1642
1643         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1644         patchcmd = diffcmd + " | git apply "
1645         tryPatchCmd = patchcmd + "--check -"
1646         applyPatchCmd = patchcmd + "--check --apply -"
1647         patch_succeeded = True
1648
1649         if os.system(tryPatchCmd) != 0:
1650             fixed_rcs_keywords = False
1651             patch_succeeded = False
1652             print "Unfortunately applying the change failed!"
1653
1654             # Patch failed, maybe it's just RCS keyword woes. Look through
1655             # the patch to see if that's possible.
1656             if gitConfigBool("git-p4.attemptRCSCleanup"):
1657                 file = None
1658                 pattern = None
1659                 kwfiles = {}
1660                 for file in editedFiles | filesToDelete:
1661                     # did this file's delta contain RCS keywords?
1662                     pattern = p4_keywords_regexp_for_file(file)
1663
1664                     if pattern:
1665                         # this file is a possibility...look for RCS keywords.
1666                         regexp = re.compile(pattern, re.VERBOSE)
1667                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1668                             if regexp.search(line):
1669                                 if verbose:
1670                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1671                                 kwfiles[file] = pattern
1672                                 break
1673
1674                 for file in kwfiles:
1675                     if verbose:
1676                         print "zapping %s with %s" % (line,pattern)
1677                     # File is being deleted, so not open in p4.  Must
1678                     # disable the read-only bit on windows.
1679                     if self.isWindows and file not in editedFiles:
1680                         os.chmod(file, stat.S_IWRITE)
1681                     self.patchRCSKeywords(file, kwfiles[file])
1682                     fixed_rcs_keywords = True
1683
1684             if fixed_rcs_keywords:
1685                 print "Retrying the patch with RCS keywords cleaned up"
1686                 if os.system(tryPatchCmd) == 0:
1687                     patch_succeeded = True
1688
1689         if not patch_succeeded:
1690             for f in editedFiles:
1691                 p4_revert(f)
1692             return False
1693
1694         #
1695         # Apply the patch for real, and do add/delete/+x handling.
1696         #
1697         system(applyPatchCmd)
1698
1699         for f in filesToChangeType:
1700             p4_edit(f, "-t", "auto")
1701         for f in filesToAdd:
1702             p4_add(f)
1703         for f in filesToDelete:
1704             p4_revert(f)
1705             p4_delete(f)
1706
1707         # Set/clear executable bits
1708         for f in filesToChangeExecBit.keys():
1709             mode = filesToChangeExecBit[f]
1710             setP4ExecBit(f, mode)
1711
1712         #
1713         # Build p4 change description, starting with the contents
1714         # of the git commit message.
1715         #
1716         logMessage = extractLogMessageFromGitCommit(id)
1717         logMessage = logMessage.strip()
1718         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1719
1720         template = self.prepareSubmitTemplate()
1721         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1722
1723         if self.preserveUser:
1724            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1725
1726         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1727             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1728             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1729             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1730
1731         separatorLine = "######## everything below this line is just the diff #######\n"
1732         if not self.prepare_p4_only:
1733             submitTemplate += separatorLine
1734             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1735
1736         (handle, fileName) = tempfile.mkstemp()
1737         tmpFile = os.fdopen(handle, "w+b")
1738         if self.isWindows:
1739             submitTemplate = submitTemplate.replace("\n", "\r\n")
1740         tmpFile.write(submitTemplate)
1741         tmpFile.close()
1742
1743         if self.prepare_p4_only:
1744             #
1745             # Leave the p4 tree prepared, and the submit template around
1746             # and let the user decide what to do next
1747             #
1748             print
1749             print "P4 workspace prepared for submission."
1750             print "To submit or revert, go to client workspace"
1751             print "  " + self.clientPath
1752             print
1753             print "To submit, use \"p4 submit\" to write a new description,"
1754             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1755                   " \"git p4\"." % fileName
1756             print "You can delete the file \"%s\" when finished." % fileName
1757
1758             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1759                 print "To preserve change ownership by user %s, you must\n" \
1760                       "do \"p4 change -f <change>\" after submitting and\n" \
1761                       "edit the User field."
1762             if pureRenameCopy:
1763                 print "After submitting, renamed files must be re-synced."
1764                 print "Invoke \"p4 sync -f\" on each of these files:"
1765                 for f in pureRenameCopy:
1766                     print "  " + f
1767
1768             print
1769             print "To revert the changes, use \"p4 revert ...\", and delete"
1770             print "the submit template file \"%s\"" % fileName
1771             if filesToAdd:
1772                 print "Since the commit adds new files, they must be deleted:"
1773                 for f in filesToAdd:
1774                     print "  " + f
1775             print
1776             return True
1777
1778         #
1779         # Let the user edit the change description, then submit it.
1780         #
1781         submitted = False
1782
1783         try:
1784             if self.edit_template(fileName):
1785                 # read the edited message and submit
1786                 tmpFile = open(fileName, "rb")
1787                 message = tmpFile.read()
1788                 tmpFile.close()
1789                 if self.isWindows:
1790                     message = message.replace("\r\n", "\n")
1791                 submitTemplate = message[:message.index(separatorLine)]
1792                 if self.shelve:
1793                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1794                 else:
1795                     p4_write_pipe(['submit', '-i'], submitTemplate)
1796                     # The rename/copy happened by applying a patch that created a
1797                     # new file.  This leaves it writable, which confuses p4.
1798                     for f in pureRenameCopy:
1799                         p4_sync(f, "-f")
1800
1801                 if self.preserveUser:
1802                     if p4User:
1803                         # Get last changelist number. Cannot easily get it from
1804                         # the submit command output as the output is
1805                         # unmarshalled.
1806                         changelist = self.lastP4Changelist()
1807                         self.modifyChangelistUser(changelist, p4User)
1808
1809                 submitted = True
1810
1811         finally:
1812             # skip this patch
1813             if not submitted or self.shelve:
1814                 if self.shelve:
1815                     print ("Reverting shelved files.")
1816                 else:
1817                     print ("Submission cancelled, undoing p4 changes.")
1818                 for f in editedFiles | filesToDelete:
1819                     p4_revert(f)
1820                 for f in filesToAdd:
1821                     p4_revert(f)
1822                     os.remove(f)
1823
1824         os.remove(fileName)
1825         return submitted
1826
1827     # Export git tags as p4 labels. Create a p4 label and then tag
1828     # with that.
1829     def exportGitTags(self, gitTags):
1830         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1831         if len(validLabelRegexp) == 0:
1832             validLabelRegexp = defaultLabelRegexp
1833         m = re.compile(validLabelRegexp)
1834
1835         for name in gitTags:
1836
1837             if not m.match(name):
1838                 if verbose:
1839                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1840                 continue
1841
1842             # Get the p4 commit this corresponds to
1843             logMessage = extractLogMessageFromGitCommit(name)
1844             values = extractSettingsGitLog(logMessage)
1845
1846             if not values.has_key('change'):
1847                 # a tag pointing to something not sent to p4; ignore
1848                 if verbose:
1849                     print "git tag %s does not give a p4 commit" % name
1850                 continue
1851             else:
1852                 changelist = values['change']
1853
1854             # Get the tag details.
1855             inHeader = True
1856             isAnnotated = False
1857             body = []
1858             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1859                 l = l.strip()
1860                 if inHeader:
1861                     if re.match(r'tag\s+', l):
1862                         isAnnotated = True
1863                     elif re.match(r'\s*$', l):
1864                         inHeader = False
1865                         continue
1866                 else:
1867                     body.append(l)
1868
1869             if not isAnnotated:
1870                 body = ["lightweight tag imported by git p4\n"]
1871
1872             # Create the label - use the same view as the client spec we are using
1873             clientSpec = getClientSpec()
1874
1875             labelTemplate  = "Label: %s\n" % name
1876             labelTemplate += "Description:\n"
1877             for b in body:
1878                 labelTemplate += "\t" + b + "\n"
1879             labelTemplate += "View:\n"
1880             for depot_side in clientSpec.mappings:
1881                 labelTemplate += "\t%s\n" % depot_side
1882
1883             if self.dry_run:
1884                 print "Would create p4 label %s for tag" % name
1885             elif self.prepare_p4_only:
1886                 print "Not creating p4 label %s for tag due to option" \
1887                       " --prepare-p4-only" % name
1888             else:
1889                 p4_write_pipe(["label", "-i"], labelTemplate)
1890
1891                 # Use the label
1892                 p4_system(["tag", "-l", name] +
1893                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1894
1895                 if verbose:
1896                     print "created p4 label for tag %s" % name
1897
1898     def run(self, args):
1899         if len(args) == 0:
1900             self.master = currentGitBranch()
1901         elif len(args) == 1:
1902             self.master = args[0]
1903             if not branchExists(self.master):
1904                 die("Branch %s does not exist" % self.master)
1905         else:
1906             return False
1907
1908         if self.master:
1909             allowSubmit = gitConfig("git-p4.allowSubmit")
1910             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1911                 die("%s is not in git-p4.allowSubmit" % self.master)
1912
1913         [upstream, settings] = findUpstreamBranchPoint()
1914         self.depotPath = settings['depot-paths'][0]
1915         if len(self.origin) == 0:
1916             self.origin = upstream
1917
1918         if self.preserveUser:
1919             if not self.canChangeChangelists():
1920                 die("Cannot preserve user names without p4 super-user or admin permissions")
1921
1922         # if not set from the command line, try the config file
1923         if self.conflict_behavior is None:
1924             val = gitConfig("git-p4.conflict")
1925             if val:
1926                 if val not in self.conflict_behavior_choices:
1927                     die("Invalid value '%s' for config git-p4.conflict" % val)
1928             else:
1929                 val = "ask"
1930             self.conflict_behavior = val
1931
1932         if self.verbose:
1933             print "Origin branch is " + self.origin
1934
1935         if len(self.depotPath) == 0:
1936             print "Internal error: cannot locate perforce depot path from existing branches"
1937             sys.exit(128)
1938
1939         self.useClientSpec = False
1940         if gitConfigBool("git-p4.useclientspec"):
1941             self.useClientSpec = True
1942         if self.useClientSpec:
1943             self.clientSpecDirs = getClientSpec()
1944
1945         # Check for the existence of P4 branches
1946         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1947
1948         if self.useClientSpec and not branchesDetected:
1949             # all files are relative to the client spec
1950             self.clientPath = getClientRoot()
1951         else:
1952             self.clientPath = p4Where(self.depotPath)
1953
1954         if self.clientPath == "":
1955             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1956
1957         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1958         self.oldWorkingDirectory = os.getcwd()
1959
1960         # ensure the clientPath exists
1961         new_client_dir = False
1962         if not os.path.exists(self.clientPath):
1963             new_client_dir = True
1964             os.makedirs(self.clientPath)
1965
1966         chdir(self.clientPath, is_client_path=True)
1967         if self.dry_run:
1968             print "Would synchronize p4 checkout in %s" % self.clientPath
1969         else:
1970             print "Synchronizing p4 checkout..."
1971             if new_client_dir:
1972                 # old one was destroyed, and maybe nobody told p4
1973                 p4_sync("...", "-f")
1974             else:
1975                 p4_sync("...")
1976         self.check()
1977
1978         commits = []
1979         if self.master:
1980             commitish = self.master
1981         else:
1982             commitish = 'HEAD'
1983
1984         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1985             commits.append(line.strip())
1986         commits.reverse()
1987
1988         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1989             self.checkAuthorship = False
1990         else:
1991             self.checkAuthorship = True
1992
1993         if self.preserveUser:
1994             self.checkValidP4Users(commits)
1995
1996         #
1997         # Build up a set of options to be passed to diff when
1998         # submitting each commit to p4.
1999         #
2000         if self.detectRenames:
2001             # command-line -M arg
2002             self.diffOpts = "-M"
2003         else:
2004             # If not explicitly set check the config variable
2005             detectRenames = gitConfig("git-p4.detectRenames")
2006
2007             if detectRenames.lower() == "false" or detectRenames == "":
2008                 self.diffOpts = ""
2009             elif detectRenames.lower() == "true":
2010                 self.diffOpts = "-M"
2011             else:
2012                 self.diffOpts = "-M%s" % detectRenames
2013
2014         # no command-line arg for -C or --find-copies-harder, just
2015         # config variables
2016         detectCopies = gitConfig("git-p4.detectCopies")
2017         if detectCopies.lower() == "false" or detectCopies == "":
2018             pass
2019         elif detectCopies.lower() == "true":
2020             self.diffOpts += " -C"
2021         else:
2022             self.diffOpts += " -C%s" % detectCopies
2023
2024         if gitConfigBool("git-p4.detectCopiesHarder"):
2025             self.diffOpts += " --find-copies-harder"
2026
2027         #
2028         # Apply the commits, one at a time.  On failure, ask if should
2029         # continue to try the rest of the patches, or quit.
2030         #
2031         if self.dry_run:
2032             print "Would apply"
2033         applied = []
2034         last = len(commits) - 1
2035         for i, commit in enumerate(commits):
2036             if self.dry_run:
2037                 print " ", read_pipe(["git", "show", "-s",
2038                                       "--format=format:%h %s", commit])
2039                 ok = True
2040             else:
2041                 ok = self.applyCommit(commit)
2042             if ok:
2043                 applied.append(commit)
2044             else:
2045                 if self.prepare_p4_only and i < last:
2046                     print "Processing only the first commit due to option" \
2047                           " --prepare-p4-only"
2048                     break
2049                 if i < last:
2050                     quit = False
2051                     while True:
2052                         # prompt for what to do, or use the option/variable
2053                         if self.conflict_behavior == "ask":
2054                             print "What do you want to do?"
2055                             response = raw_input("[s]kip this commit but apply"
2056                                                  " the rest, or [q]uit? ")
2057                             if not response:
2058                                 continue
2059                         elif self.conflict_behavior == "skip":
2060                             response = "s"
2061                         elif self.conflict_behavior == "quit":
2062                             response = "q"
2063                         else:
2064                             die("Unknown conflict_behavior '%s'" %
2065                                 self.conflict_behavior)
2066
2067                         if response[0] == "s":
2068                             print "Skipping this commit, but applying the rest"
2069                             break
2070                         if response[0] == "q":
2071                             print "Quitting"
2072                             quit = True
2073                             break
2074                     if quit:
2075                         break
2076
2077         chdir(self.oldWorkingDirectory)
2078         shelved_applied = "shelved" if self.shelve else "applied"
2079         if self.dry_run:
2080             pass
2081         elif self.prepare_p4_only:
2082             pass
2083         elif len(commits) == len(applied):
2084             print ("All commits {0}!".format(shelved_applied))
2085
2086             sync = P4Sync()
2087             if self.branch:
2088                 sync.branch = self.branch
2089             sync.run([])
2090
2091             rebase = P4Rebase()
2092             rebase.rebase()
2093
2094         else:
2095             if len(applied) == 0:
2096                 print ("No commits {0}.".format(shelved_applied))
2097             else:
2098                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2099                 for c in commits:
2100                     if c in applied:
2101                         star = "*"
2102                     else:
2103                         star = " "
2104                     print star, read_pipe(["git", "show", "-s",
2105                                            "--format=format:%h %s",  c])
2106                 print "You will have to do 'git p4 sync' and rebase."
2107
2108         if gitConfigBool("git-p4.exportLabels"):
2109             self.exportLabels = True
2110
2111         if self.exportLabels:
2112             p4Labels = getP4Labels(self.depotPath)
2113             gitTags = getGitTags()
2114
2115             missingGitTags = gitTags - p4Labels
2116             self.exportGitTags(missingGitTags)
2117
2118         # exit with error unless everything applied perfectly
2119         if len(commits) != len(applied):
2120                 sys.exit(1)
2121
2122         return True
2123
2124 class View(object):
2125     """Represent a p4 view ("p4 help views"), and map files in a
2126        repo according to the view."""
2127
2128     def __init__(self, client_name):
2129         self.mappings = []
2130         self.client_prefix = "//%s/" % client_name
2131         # cache results of "p4 where" to lookup client file locations
2132         self.client_spec_path_cache = {}
2133
2134     def append(self, view_line):
2135         """Parse a view line, splitting it into depot and client
2136            sides.  Append to self.mappings, preserving order.  This
2137            is only needed for tag creation."""
2138
2139         # Split the view line into exactly two words.  P4 enforces
2140         # structure on these lines that simplifies this quite a bit.
2141         #
2142         # Either or both words may be double-quoted.
2143         # Single quotes do not matter.
2144         # Double-quote marks cannot occur inside the words.
2145         # A + or - prefix is also inside the quotes.
2146         # There are no quotes unless they contain a space.
2147         # The line is already white-space stripped.
2148         # The two words are separated by a single space.
2149         #
2150         if view_line[0] == '"':
2151             # First word is double quoted.  Find its end.
2152             close_quote_index = view_line.find('"', 1)
2153             if close_quote_index <= 0:
2154                 die("No first-word closing quote found: %s" % view_line)
2155             depot_side = view_line[1:close_quote_index]
2156             # skip closing quote and space
2157             rhs_index = close_quote_index + 1 + 1
2158         else:
2159             space_index = view_line.find(" ")
2160             if space_index <= 0:
2161                 die("No word-splitting space found: %s" % view_line)
2162             depot_side = view_line[0:space_index]
2163             rhs_index = space_index + 1
2164
2165         # prefix + means overlay on previous mapping
2166         if depot_side.startswith("+"):
2167             depot_side = depot_side[1:]
2168
2169         # prefix - means exclude this path, leave out of mappings
2170         exclude = False
2171         if depot_side.startswith("-"):
2172             exclude = True
2173             depot_side = depot_side[1:]
2174
2175         if not exclude:
2176             self.mappings.append(depot_side)
2177
2178     def convert_client_path(self, clientFile):
2179         # chop off //client/ part to make it relative
2180         if not clientFile.startswith(self.client_prefix):
2181             die("No prefix '%s' on clientFile '%s'" %
2182                 (self.client_prefix, clientFile))
2183         return clientFile[len(self.client_prefix):]
2184
2185     def update_client_spec_path_cache(self, files):
2186         """ Caching file paths by "p4 where" batch query """
2187
2188         # List depot file paths exclude that already cached
2189         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2190
2191         if len(fileArgs) == 0:
2192             return  # All files in cache
2193
2194         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2195         for res in where_result:
2196             if "code" in res and res["code"] == "error":
2197                 # assume error is "... file(s) not in client view"
2198                 continue
2199             if "clientFile" not in res:
2200                 die("No clientFile in 'p4 where' output")
2201             if "unmap" in res:
2202                 # it will list all of them, but only one not unmap-ped
2203                 continue
2204             if gitConfigBool("core.ignorecase"):
2205                 res['depotFile'] = res['depotFile'].lower()
2206             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2207
2208         # not found files or unmap files set to ""
2209         for depotFile in fileArgs:
2210             if gitConfigBool("core.ignorecase"):
2211                 depotFile = depotFile.lower()
2212             if depotFile not in self.client_spec_path_cache:
2213                 self.client_spec_path_cache[depotFile] = ""
2214
2215     def map_in_client(self, depot_path):
2216         """Return the relative location in the client where this
2217            depot file should live.  Returns "" if the file should
2218            not be mapped in the client."""
2219
2220         if gitConfigBool("core.ignorecase"):
2221             depot_path = depot_path.lower()
2222
2223         if depot_path in self.client_spec_path_cache:
2224             return self.client_spec_path_cache[depot_path]
2225
2226         die( "Error: %s is not found in client spec path" % depot_path )
2227         return ""
2228
2229 class P4Sync(Command, P4UserMap):
2230     delete_actions = ( "delete", "move/delete", "purge" )
2231
2232     def __init__(self):
2233         Command.__init__(self)
2234         P4UserMap.__init__(self)
2235         self.options = [
2236                 optparse.make_option("--branch", dest="branch"),
2237                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2238                 optparse.make_option("--changesfile", dest="changesFile"),
2239                 optparse.make_option("--silent", dest="silent", action="store_true"),
2240                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2241                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2242                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2243                                      help="Import into refs/heads/ , not refs/remotes"),
2244                 optparse.make_option("--max-changes", dest="maxChanges",
2245                                      help="Maximum number of changes to import"),
2246                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2247                                      help="Internal block size to use when iteratively calling p4 changes"),
2248                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2249                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2250                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2251                                      help="Only sync files that are included in the Perforce Client Spec"),
2252                 optparse.make_option("-/", dest="cloneExclude",
2253                                      action="append", type="string",
2254                                      help="exclude depot path"),
2255         ]
2256         self.description = """Imports from Perforce into a git repository.\n
2257     example:
2258     //depot/my/project/ -- to import the current head
2259     //depot/my/project/@all -- to import everything
2260     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2261
2262     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2263
2264         self.usage += " //depot/path[@revRange]"
2265         self.silent = False
2266         self.createdBranches = set()
2267         self.committedChanges = set()
2268         self.branch = ""
2269         self.detectBranches = False
2270         self.detectLabels = False
2271         self.importLabels = False
2272         self.changesFile = ""
2273         self.syncWithOrigin = True
2274         self.importIntoRemotes = True
2275         self.maxChanges = ""
2276         self.changes_block_size = None
2277         self.keepRepoPath = False
2278         self.depotPaths = None
2279         self.p4BranchesInGit = []
2280         self.cloneExclude = []
2281         self.useClientSpec = False
2282         self.useClientSpec_from_options = False
2283         self.clientSpecDirs = None
2284         self.tempBranches = []
2285         self.tempBranchLocation = "refs/git-p4-tmp"
2286         self.largeFileSystem = None
2287
2288         if gitConfig('git-p4.largeFileSystem'):
2289             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2290             self.largeFileSystem = largeFileSystemConstructor(
2291                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2292             )
2293
2294         if gitConfig("git-p4.syncFromOrigin") == "false":
2295             self.syncWithOrigin = False
2296
2297     # This is required for the "append" cloneExclude action
2298     def ensure_value(self, attr, value):
2299         if not hasattr(self, attr) or getattr(self, attr) is None:
2300             setattr(self, attr, value)
2301         return getattr(self, attr)
2302
2303     # Force a checkpoint in fast-import and wait for it to finish
2304     def checkpoint(self):
2305         self.gitStream.write("checkpoint\n\n")
2306         self.gitStream.write("progress checkpoint\n\n")
2307         out = self.gitOutput.readline()
2308         if self.verbose:
2309             print "checkpoint finished: " + out
2310
2311     def extractFilesFromCommit(self, commit):
2312         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2313                              for path in self.cloneExclude]
2314         files = []
2315         fnum = 0
2316         while commit.has_key("depotFile%s" % fnum):
2317             path =  commit["depotFile%s" % fnum]
2318
2319             if [p for p in self.cloneExclude
2320                 if p4PathStartsWith(path, p)]:
2321                 found = False
2322             else:
2323                 found = [p for p in self.depotPaths
2324                          if p4PathStartsWith(path, p)]
2325             if not found:
2326                 fnum = fnum + 1
2327                 continue
2328
2329             file = {}
2330             file["path"] = path
2331             file["rev"] = commit["rev%s" % fnum]
2332             file["action"] = commit["action%s" % fnum]
2333             file["type"] = commit["type%s" % fnum]
2334             files.append(file)
2335             fnum = fnum + 1
2336         return files
2337
2338     def extractJobsFromCommit(self, commit):
2339         jobs = []
2340         jnum = 0
2341         while commit.has_key("job%s" % jnum):
2342             job = commit["job%s" % jnum]
2343             jobs.append(job)
2344             jnum = jnum + 1
2345         return jobs
2346
2347     def stripRepoPath(self, path, prefixes):
2348         """When streaming files, this is called to map a p4 depot path
2349            to where it should go in git.  The prefixes are either
2350            self.depotPaths, or self.branchPrefixes in the case of
2351            branch detection."""
2352
2353         if self.useClientSpec:
2354             # branch detection moves files up a level (the branch name)
2355             # from what client spec interpretation gives
2356             path = self.clientSpecDirs.map_in_client(path)
2357             if self.detectBranches:
2358                 for b in self.knownBranches:
2359                     if path.startswith(b + "/"):
2360                         path = path[len(b)+1:]
2361
2362         elif self.keepRepoPath:
2363             # Preserve everything in relative path name except leading
2364             # //depot/; just look at first prefix as they all should
2365             # be in the same depot.
2366             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2367             if p4PathStartsWith(path, depot):
2368                 path = path[len(depot):]
2369
2370         else:
2371             for p in prefixes:
2372                 if p4PathStartsWith(path, p):
2373                     path = path[len(p):]
2374                     break
2375
2376         path = wildcard_decode(path)
2377         return path
2378
2379     def splitFilesIntoBranches(self, commit):
2380         """Look at each depotFile in the commit to figure out to what
2381            branch it belongs."""
2382
2383         if self.clientSpecDirs:
2384             files = self.extractFilesFromCommit(commit)
2385             self.clientSpecDirs.update_client_spec_path_cache(files)
2386
2387         branches = {}
2388         fnum = 0
2389         while commit.has_key("depotFile%s" % fnum):
2390             path =  commit["depotFile%s" % fnum]
2391             found = [p for p in self.depotPaths
2392                      if p4PathStartsWith(path, p)]
2393             if not found:
2394                 fnum = fnum + 1
2395                 continue
2396
2397             file = {}
2398             file["path"] = path
2399             file["rev"] = commit["rev%s" % fnum]
2400             file["action"] = commit["action%s" % fnum]
2401             file["type"] = commit["type%s" % fnum]
2402             fnum = fnum + 1
2403
2404             # start with the full relative path where this file would
2405             # go in a p4 client
2406             if self.useClientSpec:
2407                 relPath = self.clientSpecDirs.map_in_client(path)
2408             else:
2409                 relPath = self.stripRepoPath(path, self.depotPaths)
2410
2411             for branch in self.knownBranches.keys():
2412                 # add a trailing slash so that a commit into qt/4.2foo
2413                 # doesn't end up in qt/4.2, e.g.
2414                 if relPath.startswith(branch + "/"):
2415                     if branch not in branches:
2416                         branches[branch] = []
2417                     branches[branch].append(file)
2418                     break
2419
2420         return branches
2421
2422     def writeToGitStream(self, gitMode, relPath, contents):
2423         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2424         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2425         for d in contents:
2426             self.gitStream.write(d)
2427         self.gitStream.write('\n')
2428
2429     # output one file from the P4 stream
2430     # - helper for streamP4Files
2431
2432     def streamOneP4File(self, file, contents):
2433         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2434         if verbose:
2435             size = int(self.stream_file['fileSize'])
2436             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2437             sys.stdout.flush()
2438
2439         (type_base, type_mods) = split_p4_type(file["type"])
2440
2441         git_mode = "100644"
2442         if "x" in type_mods:
2443             git_mode = "100755"
2444         if type_base == "symlink":
2445             git_mode = "120000"
2446             # p4 print on a symlink sometimes contains "target\n";
2447             # if it does, remove the newline
2448             data = ''.join(contents)
2449             if not data:
2450                 # Some version of p4 allowed creating a symlink that pointed
2451                 # to nothing.  This causes p4 errors when checking out such
2452                 # a change, and errors here too.  Work around it by ignoring
2453                 # the bad symlink; hopefully a future change fixes it.
2454                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2455                 return
2456             elif data[-1] == '\n':
2457                 contents = [data[:-1]]
2458             else:
2459                 contents = [data]
2460
2461         if type_base == "utf16":
2462             # p4 delivers different text in the python output to -G
2463             # than it does when using "print -o", or normal p4 client
2464             # operations.  utf16 is converted to ascii or utf8, perhaps.
2465             # But ascii text saved as -t utf16 is completely mangled.
2466             # Invoke print -o to get the real contents.
2467             #
2468             # On windows, the newlines will always be mangled by print, so put
2469             # them back too.  This is not needed to the cygwin windows version,
2470             # just the native "NT" type.
2471             #
2472             try:
2473                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2474             except Exception as e:
2475                 if 'Translation of file content failed' in str(e):
2476                     type_base = 'binary'
2477                 else:
2478                     raise e
2479             else:
2480                 if p4_version_string().find('/NT') >= 0:
2481                     text = text.replace('\r\n', '\n')
2482                 contents = [ text ]
2483
2484         if type_base == "apple":
2485             # Apple filetype files will be streamed as a concatenation of
2486             # its appledouble header and the contents.  This is useless
2487             # on both macs and non-macs.  If using "print -q -o xx", it
2488             # will create "xx" with the data, and "%xx" with the header.
2489             # This is also not very useful.
2490             #
2491             # Ideally, someday, this script can learn how to generate
2492             # appledouble files directly and import those to git, but
2493             # non-mac machines can never find a use for apple filetype.
2494             print "\nIgnoring apple filetype file %s" % file['depotFile']
2495             return
2496
2497         # Note that we do not try to de-mangle keywords on utf16 files,
2498         # even though in theory somebody may want that.
2499         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2500         if pattern:
2501             regexp = re.compile(pattern, re.VERBOSE)
2502             text = ''.join(contents)
2503             text = regexp.sub(r'$\1$', text)
2504             contents = [ text ]
2505
2506         try:
2507             relPath.decode('ascii')
2508         except:
2509             encoding = 'utf8'
2510             if gitConfig('git-p4.pathEncoding'):
2511                 encoding = gitConfig('git-p4.pathEncoding')
2512             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2513             if self.verbose:
2514                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2515
2516         if self.largeFileSystem:
2517             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2518
2519         self.writeToGitStream(git_mode, relPath, contents)
2520
2521     def streamOneP4Deletion(self, file):
2522         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2523         if verbose:
2524             sys.stdout.write("delete %s\n" % relPath)
2525             sys.stdout.flush()
2526         self.gitStream.write("D %s\n" % relPath)
2527
2528         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2529             self.largeFileSystem.removeLargeFile(relPath)
2530
2531     # handle another chunk of streaming data
2532     def streamP4FilesCb(self, marshalled):
2533
2534         # catch p4 errors and complain
2535         err = None
2536         if "code" in marshalled:
2537             if marshalled["code"] == "error":
2538                 if "data" in marshalled:
2539                     err = marshalled["data"].rstrip()
2540
2541         if not err and 'fileSize' in self.stream_file:
2542             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2543             if required_bytes > 0:
2544                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2545                     os.getcwd(), required_bytes/1024/1024
2546                 )
2547
2548         if err:
2549             f = None
2550             if self.stream_have_file_info:
2551                 if "depotFile" in self.stream_file:
2552                     f = self.stream_file["depotFile"]
2553             # force a failure in fast-import, else an empty
2554             # commit will be made
2555             self.gitStream.write("\n")
2556             self.gitStream.write("die-now\n")
2557             self.gitStream.close()
2558             # ignore errors, but make sure it exits first
2559             self.importProcess.wait()
2560             if f:
2561                 die("Error from p4 print for %s: %s" % (f, err))
2562             else:
2563                 die("Error from p4 print: %s" % err)
2564
2565         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2566             # start of a new file - output the old one first
2567             self.streamOneP4File(self.stream_file, self.stream_contents)
2568             self.stream_file = {}
2569             self.stream_contents = []
2570             self.stream_have_file_info = False
2571
2572         # pick up the new file information... for the
2573         # 'data' field we need to append to our array
2574         for k in marshalled.keys():
2575             if k == 'data':
2576                 if 'streamContentSize' not in self.stream_file:
2577                     self.stream_file['streamContentSize'] = 0
2578                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2579                 self.stream_contents.append(marshalled['data'])
2580             else:
2581                 self.stream_file[k] = marshalled[k]
2582
2583         if (verbose and
2584             'streamContentSize' in self.stream_file and
2585             'fileSize' in self.stream_file and
2586             'depotFile' in self.stream_file):
2587             size = int(self.stream_file["fileSize"])
2588             if size > 0:
2589                 progress = 100*self.stream_file['streamContentSize']/size
2590                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2591                 sys.stdout.flush()
2592
2593         self.stream_have_file_info = True
2594
2595     # Stream directly from "p4 files" into "git fast-import"
2596     def streamP4Files(self, files):
2597         filesForCommit = []
2598         filesToRead = []
2599         filesToDelete = []
2600
2601         for f in files:
2602             filesForCommit.append(f)
2603             if f['action'] in self.delete_actions:
2604                 filesToDelete.append(f)
2605             else:
2606                 filesToRead.append(f)
2607
2608         # deleted files...
2609         for f in filesToDelete:
2610             self.streamOneP4Deletion(f)
2611
2612         if len(filesToRead) > 0:
2613             self.stream_file = {}
2614             self.stream_contents = []
2615             self.stream_have_file_info = False
2616
2617             # curry self argument
2618             def streamP4FilesCbSelf(entry):
2619                 self.streamP4FilesCb(entry)
2620
2621             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2622
2623             p4CmdList(["-x", "-", "print"],
2624                       stdin=fileArgs,
2625                       cb=streamP4FilesCbSelf)
2626
2627             # do the last chunk
2628             if self.stream_file.has_key('depotFile'):
2629                 self.streamOneP4File(self.stream_file, self.stream_contents)
2630
2631     def make_email(self, userid):
2632         if userid in self.users:
2633             return self.users[userid]
2634         else:
2635             return "%s <a@b>" % userid
2636
2637     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2638         """ Stream a p4 tag.
2639         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2640         """
2641
2642         if verbose:
2643             print "writing tag %s for commit %s" % (labelName, commit)
2644         gitStream.write("tag %s\n" % labelName)
2645         gitStream.write("from %s\n" % commit)
2646
2647         if labelDetails.has_key('Owner'):
2648             owner = labelDetails["Owner"]
2649         else:
2650             owner = None
2651
2652         # Try to use the owner of the p4 label, or failing that,
2653         # the current p4 user id.
2654         if owner:
2655             email = self.make_email(owner)
2656         else:
2657             email = self.make_email(self.p4UserId())
2658         tagger = "%s %s %s" % (email, epoch, self.tz)
2659
2660         gitStream.write("tagger %s\n" % tagger)
2661
2662         print "labelDetails=",labelDetails
2663         if labelDetails.has_key('Description'):
2664             description = labelDetails['Description']
2665         else:
2666             description = 'Label from git p4'
2667
2668         gitStream.write("data %d\n" % len(description))
2669         gitStream.write(description)
2670         gitStream.write("\n")
2671
2672     def inClientSpec(self, path):
2673         if not self.clientSpecDirs:
2674             return True
2675         inClientSpec = self.clientSpecDirs.map_in_client(path)
2676         if not inClientSpec and self.verbose:
2677             print('Ignoring file outside of client spec: {0}'.format(path))
2678         return inClientSpec
2679
2680     def hasBranchPrefix(self, path):
2681         if not self.branchPrefixes:
2682             return True
2683         hasPrefix = [p for p in self.branchPrefixes
2684                         if p4PathStartsWith(path, p)]
2685         if not hasPrefix and self.verbose:
2686             print('Ignoring file outside of prefix: {0}'.format(path))
2687         return hasPrefix
2688
2689     def commit(self, details, files, branch, parent = ""):
2690         epoch = details["time"]
2691         author = details["user"]
2692         jobs = self.extractJobsFromCommit(details)
2693
2694         if self.verbose:
2695             print('commit into {0}'.format(branch))
2696
2697         if self.clientSpecDirs:
2698             self.clientSpecDirs.update_client_spec_path_cache(files)
2699
2700         files = [f for f in files
2701             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2702
2703         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2704             print('Ignoring revision {0} as it would produce an empty commit.'
2705                 .format(details['change']))
2706             return
2707
2708         self.gitStream.write("commit %s\n" % branch)
2709         self.gitStream.write("mark :%s\n" % details["change"])
2710         self.committedChanges.add(int(details["change"]))
2711         committer = ""
2712         if author not in self.users:
2713             self.getUserMapFromPerforceServer()
2714         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2715
2716         self.gitStream.write("committer %s\n" % committer)
2717
2718         self.gitStream.write("data <<EOT\n")
2719         self.gitStream.write(details["desc"])
2720         if len(jobs) > 0:
2721             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2722         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2723                              (','.join(self.branchPrefixes), details["change"]))
2724         if len(details['options']) > 0:
2725             self.gitStream.write(": options = %s" % details['options'])
2726         self.gitStream.write("]\nEOT\n\n")
2727
2728         if len(parent) > 0:
2729             if self.verbose:
2730                 print "parent %s" % parent
2731             self.gitStream.write("from %s\n" % parent)
2732
2733         self.streamP4Files(files)
2734         self.gitStream.write("\n")
2735
2736         change = int(details["change"])
2737
2738         if self.labels.has_key(change):
2739             label = self.labels[change]
2740             labelDetails = label[0]
2741             labelRevisions = label[1]
2742             if self.verbose:
2743                 print "Change %s is labelled %s" % (change, labelDetails)
2744
2745             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2746                                                 for p in self.branchPrefixes])
2747
2748             if len(files) == len(labelRevisions):
2749
2750                 cleanedFiles = {}
2751                 for info in files:
2752                     if info["action"] in self.delete_actions:
2753                         continue
2754                     cleanedFiles[info["depotFile"]] = info["rev"]
2755
2756                 if cleanedFiles == labelRevisions:
2757                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2758
2759                 else:
2760                     if not self.silent:
2761                         print ("Tag %s does not match with change %s: files do not match."
2762                                % (labelDetails["label"], change))
2763
2764             else:
2765                 if not self.silent:
2766                     print ("Tag %s does not match with change %s: file count is different."
2767                            % (labelDetails["label"], change))
2768
2769     # Build a dictionary of changelists and labels, for "detect-labels" option.
2770     def getLabels(self):
2771         self.labels = {}
2772
2773         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2774         if len(l) > 0 and not self.silent:
2775             print "Finding files belonging to labels in %s" % `self.depotPaths`
2776
2777         for output in l:
2778             label = output["label"]
2779             revisions = {}
2780             newestChange = 0
2781             if self.verbose:
2782                 print "Querying files for label %s" % label
2783             for file in p4CmdList(["files"] +
2784                                       ["%s...@%s" % (p, label)
2785                                           for p in self.depotPaths]):
2786                 revisions[file["depotFile"]] = file["rev"]
2787                 change = int(file["change"])
2788                 if change > newestChange:
2789                     newestChange = change
2790
2791             self.labels[newestChange] = [output, revisions]
2792
2793         if self.verbose:
2794             print "Label changes: %s" % self.labels.keys()
2795
2796     # Import p4 labels as git tags. A direct mapping does not
2797     # exist, so assume that if all the files are at the same revision
2798     # then we can use that, or it's something more complicated we should
2799     # just ignore.
2800     def importP4Labels(self, stream, p4Labels):
2801         if verbose:
2802             print "import p4 labels: " + ' '.join(p4Labels)
2803
2804         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2805         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2806         if len(validLabelRegexp) == 0:
2807             validLabelRegexp = defaultLabelRegexp
2808         m = re.compile(validLabelRegexp)
2809
2810         for name in p4Labels:
2811             commitFound = False
2812
2813             if not m.match(name):
2814                 if verbose:
2815                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2816                 continue
2817
2818             if name in ignoredP4Labels:
2819                 continue
2820
2821             labelDetails = p4CmdList(['label', "-o", name])[0]
2822
2823             # get the most recent changelist for each file in this label
2824             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2825                                 for p in self.depotPaths])
2826
2827             if change.has_key('change'):
2828                 # find the corresponding git commit; take the oldest commit
2829                 changelist = int(change['change'])
2830                 if changelist in self.committedChanges:
2831                     gitCommit = ":%d" % changelist       # use a fast-import mark
2832                     commitFound = True
2833                 else:
2834                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2835                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2836                     if len(gitCommit) == 0:
2837                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2838                     else:
2839                         commitFound = True
2840                         gitCommit = gitCommit.strip()
2841
2842                 if commitFound:
2843                     # Convert from p4 time format
2844                     try:
2845                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2846                     except ValueError:
2847                         print "Could not convert label time %s" % labelDetails['Update']
2848                         tmwhen = 1
2849
2850                     when = int(time.mktime(tmwhen))
2851                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2852                     if verbose:
2853                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2854             else:
2855                 if verbose:
2856                     print "Label %s has no changelists - possibly deleted?" % name
2857
2858             if not commitFound:
2859                 # We can't import this label; don't try again as it will get very
2860                 # expensive repeatedly fetching all the files for labels that will
2861                 # never be imported. If the label is moved in the future, the
2862                 # ignore will need to be removed manually.
2863                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2864
2865     def guessProjectName(self):
2866         for p in self.depotPaths:
2867             if p.endswith("/"):
2868                 p = p[:-1]
2869             p = p[p.strip().rfind("/") + 1:]
2870             if not p.endswith("/"):
2871                p += "/"
2872             return p
2873
2874     def getBranchMapping(self):
2875         lostAndFoundBranches = set()
2876
2877         user = gitConfig("git-p4.branchUser")
2878         if len(user) > 0:
2879             command = "branches -u %s" % user
2880         else:
2881             command = "branches"
2882
2883         for info in p4CmdList(command):
2884             details = p4Cmd(["branch", "-o", info["branch"]])
2885             viewIdx = 0
2886             while details.has_key("View%s" % viewIdx):
2887                 paths = details["View%s" % viewIdx].split(" ")
2888                 viewIdx = viewIdx + 1
2889                 # require standard //depot/foo/... //depot/bar/... mapping
2890                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2891                     continue
2892                 source = paths[0]
2893                 destination = paths[1]
2894                 ## HACK
2895                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2896                     source = source[len(self.depotPaths[0]):-4]
2897                     destination = destination[len(self.depotPaths[0]):-4]
2898
2899                     if destination in self.knownBranches:
2900                         if not self.silent:
2901                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2902                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2903                         continue
2904
2905                     self.knownBranches[destination] = source
2906
2907                     lostAndFoundBranches.discard(destination)
2908
2909                     if source not in self.knownBranches:
2910                         lostAndFoundBranches.add(source)
2911
2912         # Perforce does not strictly require branches to be defined, so we also
2913         # check git config for a branch list.
2914         #
2915         # Example of branch definition in git config file:
2916         # [git-p4]
2917         #   branchList=main:branchA
2918         #   branchList=main:branchB
2919         #   branchList=branchA:branchC
2920         configBranches = gitConfigList("git-p4.branchList")
2921         for branch in configBranches:
2922             if branch:
2923                 (source, destination) = branch.split(":")
2924                 self.knownBranches[destination] = source
2925
2926                 lostAndFoundBranches.discard(destination)
2927
2928                 if source not in self.knownBranches:
2929                     lostAndFoundBranches.add(source)
2930
2931
2932         for branch in lostAndFoundBranches:
2933             self.knownBranches[branch] = branch
2934
2935     def getBranchMappingFromGitBranches(self):
2936         branches = p4BranchesInGit(self.importIntoRemotes)
2937         for branch in branches.keys():
2938             if branch == "master":
2939                 branch = "main"
2940             else:
2941                 branch = branch[len(self.projectName):]
2942             self.knownBranches[branch] = branch
2943
2944     def updateOptionDict(self, d):
2945         option_keys = {}
2946         if self.keepRepoPath:
2947             option_keys['keepRepoPath'] = 1
2948
2949         d["options"] = ' '.join(sorted(option_keys.keys()))
2950
2951     def readOptions(self, d):
2952         self.keepRepoPath = (d.has_key('options')
2953                              and ('keepRepoPath' in d['options']))
2954
2955     def gitRefForBranch(self, branch):
2956         if branch == "main":
2957             return self.refPrefix + "master"
2958
2959         if len(branch) <= 0:
2960             return branch
2961
2962         return self.refPrefix + self.projectName + branch
2963
2964     def gitCommitByP4Change(self, ref, change):
2965         if self.verbose:
2966             print "looking in ref " + ref + " for change %s using bisect..." % change
2967
2968         earliestCommit = ""
2969         latestCommit = parseRevision(ref)
2970
2971         while True:
2972             if self.verbose:
2973                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2974             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2975             if len(next) == 0:
2976                 if self.verbose:
2977                     print "argh"
2978                 return ""
2979             log = extractLogMessageFromGitCommit(next)
2980             settings = extractSettingsGitLog(log)
2981             currentChange = int(settings['change'])
2982             if self.verbose:
2983                 print "current change %s" % currentChange
2984
2985             if currentChange == change:
2986                 if self.verbose:
2987                     print "found %s" % next
2988                 return next
2989
2990             if currentChange < change:
2991                 earliestCommit = "^%s" % next
2992             else:
2993                 latestCommit = "%s" % next
2994
2995         return ""
2996
2997     def importNewBranch(self, branch, maxChange):
2998         # make fast-import flush all changes to disk and update the refs using the checkpoint
2999         # command so that we can try to find the branch parent in the git history
3000         self.gitStream.write("checkpoint\n\n");
3001         self.gitStream.flush();
3002         branchPrefix = self.depotPaths[0] + branch + "/"
3003         range = "@1,%s" % maxChange
3004         #print "prefix" + branchPrefix
3005         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3006         if len(changes) <= 0:
3007             return False
3008         firstChange = changes[0]
3009         #print "first change in branch: %s" % firstChange
3010         sourceBranch = self.knownBranches[branch]
3011         sourceDepotPath = self.depotPaths[0] + sourceBranch
3012         sourceRef = self.gitRefForBranch(sourceBranch)
3013         #print "source " + sourceBranch
3014
3015         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3016         #print "branch parent: %s" % branchParentChange
3017         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3018         if len(gitParent) > 0:
3019             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3020             #print "parent git commit: %s" % gitParent
3021
3022         self.importChanges(changes)
3023         return True
3024
3025     def searchParent(self, parent, branch, target):
3026         parentFound = False
3027         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3028                                      "--no-merges", parent]):
3029             blob = blob.strip()
3030             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3031                 parentFound = True
3032                 if self.verbose:
3033                     print "Found parent of %s in commit %s" % (branch, blob)
3034                 break
3035         if parentFound:
3036             return blob
3037         else:
3038             return None
3039
3040     def importChanges(self, changes):
3041         cnt = 1
3042         for change in changes:
3043             description = p4_describe(change)
3044             self.updateOptionDict(description)
3045
3046             if not self.silent:
3047                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3048                 sys.stdout.flush()
3049             cnt = cnt + 1
3050
3051             try:
3052                 if self.detectBranches:
3053                     branches = self.splitFilesIntoBranches(description)
3054                     for branch in branches.keys():
3055                         ## HACK  --hwn
3056                         branchPrefix = self.depotPaths[0] + branch + "/"
3057                         self.branchPrefixes = [ branchPrefix ]
3058
3059                         parent = ""
3060
3061                         filesForCommit = branches[branch]
3062
3063                         if self.verbose:
3064                             print "branch is %s" % branch
3065
3066                         self.updatedBranches.add(branch)
3067
3068                         if branch not in self.createdBranches:
3069                             self.createdBranches.add(branch)
3070                             parent = self.knownBranches[branch]
3071                             if parent == branch:
3072                                 parent = ""
3073                             else:
3074                                 fullBranch = self.projectName + branch
3075                                 if fullBranch not in self.p4BranchesInGit:
3076                                     if not self.silent:
3077                                         print("\n    Importing new branch %s" % fullBranch);
3078                                     if self.importNewBranch(branch, change - 1):
3079                                         parent = ""
3080                                         self.p4BranchesInGit.append(fullBranch)
3081                                     if not self.silent:
3082                                         print("\n    Resuming with change %s" % change);
3083
3084                                 if self.verbose:
3085                                     print "parent determined through known branches: %s" % parent
3086
3087                         branch = self.gitRefForBranch(branch)
3088                         parent = self.gitRefForBranch(parent)
3089
3090                         if self.verbose:
3091                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3092
3093                         if len(parent) == 0 and branch in self.initialParents:
3094                             parent = self.initialParents[branch]
3095                             del self.initialParents[branch]
3096
3097                         blob = None
3098                         if len(parent) > 0:
3099                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3100                             if self.verbose:
3101                                 print "Creating temporary branch: " + tempBranch
3102                             self.commit(description, filesForCommit, tempBranch)
3103                             self.tempBranches.append(tempBranch)
3104                             self.checkpoint()
3105                             blob = self.searchParent(parent, branch, tempBranch)
3106                         if blob:
3107                             self.commit(description, filesForCommit, branch, blob)
3108                         else:
3109                             if self.verbose:
3110                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3111                             self.commit(description, filesForCommit, branch, parent)
3112                 else:
3113                     files = self.extractFilesFromCommit(description)
3114                     self.commit(description, files, self.branch,
3115                                 self.initialParent)
3116                     # only needed once, to connect to the previous commit
3117                     self.initialParent = ""
3118             except IOError:
3119                 print self.gitError.read()
3120                 sys.exit(1)
3121
3122     def importHeadRevision(self, revision):
3123         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3124
3125         details = {}
3126         details["user"] = "git perforce import user"
3127         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3128                            % (' '.join(self.depotPaths), revision))
3129         details["change"] = revision
3130         newestRevision = 0
3131
3132         fileCnt = 0
3133         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3134
3135         for info in p4CmdList(["files"] + fileArgs):
3136
3137             if 'code' in info and info['code'] == 'error':
3138                 sys.stderr.write("p4 returned an error: %s\n"
3139                                  % info['data'])
3140                 if info['data'].find("must refer to client") >= 0:
3141                     sys.stderr.write("This particular p4 error is misleading.\n")
3142                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3143                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3144                 sys.exit(1)
3145             if 'p4ExitCode' in info:
3146                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3147                 sys.exit(1)
3148
3149
3150             change = int(info["change"])
3151             if change > newestRevision:
3152                 newestRevision = change
3153
3154             if info["action"] in self.delete_actions:
3155                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3156                 #fileCnt = fileCnt + 1
3157                 continue
3158
3159             for prop in ["depotFile", "rev", "action", "type" ]:
3160                 details["%s%s" % (prop, fileCnt)] = info[prop]
3161
3162             fileCnt = fileCnt + 1
3163
3164         details["change"] = newestRevision
3165
3166         # Use time from top-most change so that all git p4 clones of
3167         # the same p4 repo have the same commit SHA1s.
3168         res = p4_describe(newestRevision)
3169         details["time"] = res["time"]
3170
3171         self.updateOptionDict(details)
3172         try:
3173             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3174         except IOError:
3175             print "IO error with git fast-import. Is your git version recent enough?"
3176             print self.gitError.read()
3177
3178
3179     def run(self, args):
3180         self.depotPaths = []
3181         self.changeRange = ""
3182         self.previousDepotPaths = []
3183         self.hasOrigin = False
3184
3185         # map from branch depot path to parent branch
3186         self.knownBranches = {}
3187         self.initialParents = {}
3188
3189         if self.importIntoRemotes:
3190             self.refPrefix = "refs/remotes/p4/"
3191         else:
3192             self.refPrefix = "refs/heads/p4/"
3193
3194         if self.syncWithOrigin:
3195             self.hasOrigin = originP4BranchesExist()
3196             if self.hasOrigin:
3197                 if not self.silent:
3198                     print 'Syncing with origin first, using "git fetch origin"'
3199                 system("git fetch origin")
3200
3201         branch_arg_given = bool(self.branch)
3202         if len(self.branch) == 0:
3203             self.branch = self.refPrefix + "master"
3204             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3205                 system("git update-ref %s refs/heads/p4" % self.branch)
3206                 system("git branch -D p4")
3207
3208         # accept either the command-line option, or the configuration variable
3209         if self.useClientSpec:
3210             # will use this after clone to set the variable
3211             self.useClientSpec_from_options = True
3212         else:
3213             if gitConfigBool("git-p4.useclientspec"):
3214                 self.useClientSpec = True
3215         if self.useClientSpec:
3216             self.clientSpecDirs = getClientSpec()
3217
3218         # TODO: should always look at previous commits,
3219         # merge with previous imports, if possible.
3220         if args == []:
3221             if self.hasOrigin:
3222                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3223
3224             # branches holds mapping from branch name to sha1
3225             branches = p4BranchesInGit(self.importIntoRemotes)
3226
3227             # restrict to just this one, disabling detect-branches
3228             if branch_arg_given:
3229                 short = self.branch.split("/")[-1]
3230                 if short in branches:
3231                     self.p4BranchesInGit = [ short ]
3232             else:
3233                 self.p4BranchesInGit = branches.keys()
3234
3235             if len(self.p4BranchesInGit) > 1:
3236                 if not self.silent:
3237                     print "Importing from/into multiple branches"
3238                 self.detectBranches = True
3239                 for branch in branches.keys():
3240                     self.initialParents[self.refPrefix + branch] = \
3241                         branches[branch]
3242
3243             if self.verbose:
3244                 print "branches: %s" % self.p4BranchesInGit
3245
3246             p4Change = 0
3247             for branch in self.p4BranchesInGit:
3248                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3249
3250                 settings = extractSettingsGitLog(logMsg)
3251
3252                 self.readOptions(settings)
3253                 if (settings.has_key('depot-paths')
3254                     and settings.has_key ('change')):
3255                     change = int(settings['change']) + 1
3256                     p4Change = max(p4Change, change)
3257
3258                     depotPaths = sorted(settings['depot-paths'])
3259                     if self.previousDepotPaths == []:
3260                         self.previousDepotPaths = depotPaths
3261                     else:
3262                         paths = []
3263                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3264                             prev_list = prev.split("/")
3265                             cur_list = cur.split("/")
3266                             for i in range(0, min(len(cur_list), len(prev_list))):
3267                                 if cur_list[i] <> prev_list[i]:
3268                                     i = i - 1
3269                                     break
3270
3271                             paths.append ("/".join(cur_list[:i + 1]))
3272
3273                         self.previousDepotPaths = paths
3274
3275             if p4Change > 0:
3276                 self.depotPaths = sorted(self.previousDepotPaths)
3277                 self.changeRange = "@%s,#head" % p4Change
3278                 if not self.silent and not self.detectBranches:
3279                     print "Performing incremental import into %s git branch" % self.branch
3280
3281         # accept multiple ref name abbreviations:
3282         #    refs/foo/bar/branch -> use it exactly
3283         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3284         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3285         if not self.branch.startswith("refs/"):
3286             if self.importIntoRemotes:
3287                 prepend = "refs/remotes/"
3288             else:
3289                 prepend = "refs/heads/"
3290             if not self.branch.startswith("p4/"):
3291                 prepend += "p4/"
3292             self.branch = prepend + self.branch
3293
3294         if len(args) == 0 and self.depotPaths:
3295             if not self.silent:
3296                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3297         else:
3298             if self.depotPaths and self.depotPaths != args:
3299                 print ("previous import used depot path %s and now %s was specified. "
3300                        "This doesn't work!" % (' '.join (self.depotPaths),
3301                                                ' '.join (args)))
3302                 sys.exit(1)
3303
3304             self.depotPaths = sorted(args)
3305
3306         revision = ""
3307         self.users = {}
3308
3309         # Make sure no revision specifiers are used when --changesfile
3310         # is specified.
3311         bad_changesfile = False
3312         if len(self.changesFile) > 0:
3313             for p in self.depotPaths:
3314                 if p.find("@") >= 0 or p.find("#") >= 0:
3315                     bad_changesfile = True
3316                     break
3317         if bad_changesfile:
3318             die("Option --changesfile is incompatible with revision specifiers")
3319
3320         newPaths = []
3321         for p in self.depotPaths:
3322             if p.find("@") != -1:
3323                 atIdx = p.index("@")
3324                 self.changeRange = p[atIdx:]
3325                 if self.changeRange == "@all":
3326                     self.changeRange = ""
3327                 elif ',' not in self.changeRange:
3328                     revision = self.changeRange
3329                     self.changeRange = ""
3330                 p = p[:atIdx]
3331             elif p.find("#") != -1:
3332                 hashIdx = p.index("#")
3333                 revision = p[hashIdx:]
3334                 p = p[:hashIdx]
3335             elif self.previousDepotPaths == []:
3336                 # pay attention to changesfile, if given, else import
3337                 # the entire p4 tree at the head revision
3338                 if len(self.changesFile) == 0:
3339                     revision = "#head"
3340
3341             p = re.sub ("\.\.\.$", "", p)
3342             if not p.endswith("/"):
3343                 p += "/"
3344
3345             newPaths.append(p)
3346
3347         self.depotPaths = newPaths
3348
3349         # --detect-branches may change this for each branch
3350         self.branchPrefixes = self.depotPaths
3351
3352         self.loadUserMapFromCache()
3353         self.labels = {}
3354         if self.detectLabels:
3355             self.getLabels();
3356
3357         if self.detectBranches:
3358             ## FIXME - what's a P4 projectName ?
3359             self.projectName = self.guessProjectName()
3360
3361             if self.hasOrigin:
3362                 self.getBranchMappingFromGitBranches()
3363             else:
3364                 self.getBranchMapping()
3365             if self.verbose:
3366                 print "p4-git branches: %s" % self.p4BranchesInGit
3367                 print "initial parents: %s" % self.initialParents
3368             for b in self.p4BranchesInGit:
3369                 if b != "master":
3370
3371                     ## FIXME
3372                     b = b[len(self.projectName):]
3373                 self.createdBranches.add(b)
3374
3375         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3376
3377         self.importProcess = subprocess.Popen(["git", "fast-import"],
3378                                               stdin=subprocess.PIPE,
3379                                               stdout=subprocess.PIPE,
3380                                               stderr=subprocess.PIPE);
3381         self.gitOutput = self.importProcess.stdout
3382         self.gitStream = self.importProcess.stdin
3383         self.gitError = self.importProcess.stderr
3384
3385         if revision:
3386             self.importHeadRevision(revision)
3387         else:
3388             changes = []
3389
3390             if len(self.changesFile) > 0:
3391                 output = open(self.changesFile).readlines()
3392                 changeSet = set()
3393                 for line in output:
3394                     changeSet.add(int(line))
3395
3396                 for change in changeSet:
3397                     changes.append(change)
3398
3399                 changes.sort()
3400             else:
3401                 # catch "git p4 sync" with no new branches, in a repo that
3402                 # does not have any existing p4 branches
3403                 if len(args) == 0:
3404                     if not self.p4BranchesInGit:
3405                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3406
3407                     # The default branch is master, unless --branch is used to
3408                     # specify something else.  Make sure it exists, or complain
3409                     # nicely about how to use --branch.
3410                     if not self.detectBranches:
3411                         if not branch_exists(self.branch):
3412                             if branch_arg_given:
3413                                 die("Error: branch %s does not exist." % self.branch)
3414                             else:
3415                                 die("Error: no branch %s; perhaps specify one with --branch." %
3416                                     self.branch)
3417
3418                 if self.verbose:
3419                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3420                                                               self.changeRange)
3421                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3422
3423                 if len(self.maxChanges) > 0:
3424                     changes = changes[:min(int(self.maxChanges), len(changes))]
3425
3426             if len(changes) == 0:
3427                 if not self.silent:
3428                     print "No changes to import!"
3429             else:
3430                 if not self.silent and not self.detectBranches:
3431                     print "Import destination: %s" % self.branch
3432
3433                 self.updatedBranches = set()
3434
3435                 if not self.detectBranches:
3436                     if args:
3437                         # start a new branch
3438                         self.initialParent = ""
3439                     else:
3440                         # build on a previous revision
3441                         self.initialParent = parseRevision(self.branch)
3442
3443                 self.importChanges(changes)
3444
3445                 if not self.silent:
3446                     print ""
3447                     if len(self.updatedBranches) > 0:
3448                         sys.stdout.write("Updated branches: ")
3449                         for b in self.updatedBranches:
3450                             sys.stdout.write("%s " % b)
3451                         sys.stdout.write("\n")
3452
3453         if gitConfigBool("git-p4.importLabels"):
3454             self.importLabels = True
3455
3456         if self.importLabels:
3457             p4Labels = getP4Labels(self.depotPaths)
3458             gitTags = getGitTags()
3459
3460             missingP4Labels = p4Labels - gitTags
3461             self.importP4Labels(self.gitStream, missingP4Labels)
3462
3463         self.gitStream.close()
3464         if self.importProcess.wait() != 0:
3465             die("fast-import failed: %s" % self.gitError.read())
3466         self.gitOutput.close()
3467         self.gitError.close()
3468
3469         # Cleanup temporary branches created during import
3470         if self.tempBranches != []:
3471             for branch in self.tempBranches:
3472                 read_pipe("git update-ref -d %s" % branch)
3473             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3474
3475         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3476         # a convenient shortcut refname "p4".
3477         if self.importIntoRemotes:
3478             head_ref = self.refPrefix + "HEAD"
3479             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3480                 system(["git", "symbolic-ref", head_ref, self.branch])
3481
3482         return True
3483
3484 class P4Rebase(Command):
3485     def __init__(self):
3486         Command.__init__(self)
3487         self.options = [
3488                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3489         ]
3490         self.importLabels = False
3491         self.description = ("Fetches the latest revision from perforce and "
3492                             + "rebases the current work (branch) against it")
3493
3494     def run(self, args):
3495         sync = P4Sync()
3496         sync.importLabels = self.importLabels
3497         sync.run([])
3498
3499         return self.rebase()
3500
3501     def rebase(self):
3502         if os.system("git update-index --refresh") != 0:
3503             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3504         if len(read_pipe("git diff-index HEAD --")) > 0:
3505             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3506
3507         [upstream, settings] = findUpstreamBranchPoint()
3508         if len(upstream) == 0:
3509             die("Cannot find upstream branchpoint for rebase")
3510
3511         # the branchpoint may be p4/foo~3, so strip off the parent
3512         upstream = re.sub("~[0-9]+$", "", upstream)
3513
3514         print "Rebasing the current branch onto %s" % upstream
3515         oldHead = read_pipe("git rev-parse HEAD").strip()
3516         system("git rebase %s" % upstream)
3517         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3518         return True
3519
3520 class P4Clone(P4Sync):
3521     def __init__(self):
3522         P4Sync.__init__(self)
3523         self.description = "Creates a new git repository and imports from Perforce into it"
3524         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3525         self.options += [
3526             optparse.make_option("--destination", dest="cloneDestination",
3527                                  action='store', default=None,
3528                                  help="where to leave result of the clone"),
3529             optparse.make_option("--bare", dest="cloneBare",
3530                                  action="store_true", default=False),
3531         ]
3532         self.cloneDestination = None
3533         self.needsGit = False
3534         self.cloneBare = False
3535
3536     def defaultDestination(self, args):
3537         ## TODO: use common prefix of args?
3538         depotPath = args[0]
3539         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3540         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3541         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3542         depotDir = re.sub(r"/$", "", depotDir)
3543         return os.path.split(depotDir)[1]
3544
3545     def run(self, args):
3546         if len(args) < 1:
3547             return False
3548
3549         if self.keepRepoPath and not self.cloneDestination:
3550             sys.stderr.write("Must specify destination for --keep-path\n")
3551             sys.exit(1)
3552
3553         depotPaths = args
3554
3555         if not self.cloneDestination and len(depotPaths) > 1:
3556             self.cloneDestination = depotPaths[-1]
3557             depotPaths = depotPaths[:-1]
3558
3559         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3560         for p in depotPaths:
3561             if not p.startswith("//"):
3562                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3563                 return False
3564
3565         if not self.cloneDestination:
3566             self.cloneDestination = self.defaultDestination(args)
3567
3568         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3569
3570         if not os.path.exists(self.cloneDestination):
3571             os.makedirs(self.cloneDestination)
3572         chdir(self.cloneDestination)
3573
3574         init_cmd = [ "git", "init" ]
3575         if self.cloneBare:
3576             init_cmd.append("--bare")
3577         retcode = subprocess.call(init_cmd)
3578         if retcode:
3579             raise CalledProcessError(retcode, init_cmd)
3580
3581         if not P4Sync.run(self, depotPaths):
3582             return False
3583
3584         # create a master branch and check out a work tree
3585         if gitBranchExists(self.branch):
3586             system([ "git", "branch", "master", self.branch ])
3587             if not self.cloneBare:
3588                 system([ "git", "checkout", "-f" ])
3589         else:
3590             print 'Not checking out any branch, use ' \
3591                   '"git checkout -q -b master <branch>"'
3592
3593         # auto-set this variable if invoked with --use-client-spec
3594         if self.useClientSpec_from_options:
3595             system("git config --bool git-p4.useclientspec true")
3596
3597         return True
3598
3599 class P4Branches(Command):
3600     def __init__(self):
3601         Command.__init__(self)
3602         self.options = [ ]
3603         self.description = ("Shows the git branches that hold imports and their "
3604                             + "corresponding perforce depot paths")
3605         self.verbose = False
3606
3607     def run(self, args):
3608         if originP4BranchesExist():
3609             createOrUpdateBranchesFromOrigin()
3610
3611         cmdline = "git rev-parse --symbolic "
3612         cmdline += " --remotes"
3613
3614         for line in read_pipe_lines(cmdline):
3615             line = line.strip()
3616
3617             if not line.startswith('p4/') or line == "p4/HEAD":
3618                 continue
3619             branch = line
3620
3621             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3622             settings = extractSettingsGitLog(log)
3623
3624             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3625         return True
3626
3627 class HelpFormatter(optparse.IndentedHelpFormatter):
3628     def __init__(self):
3629         optparse.IndentedHelpFormatter.__init__(self)
3630
3631     def format_description(self, description):
3632         if description:
3633             return description + "\n"
3634         else:
3635             return ""
3636
3637 def printUsage(commands):
3638     print "usage: %s <command> [options]" % sys.argv[0]
3639     print ""
3640     print "valid commands: %s" % ", ".join(commands)
3641     print ""
3642     print "Try %s <command> --help for command specific help." % sys.argv[0]
3643     print ""
3644
3645 commands = {
3646     "debug" : P4Debug,
3647     "submit" : P4Submit,
3648     "commit" : P4Submit,
3649     "sync" : P4Sync,
3650     "rebase" : P4Rebase,
3651     "clone" : P4Clone,
3652     "rollback" : P4RollBack,
3653     "branches" : P4Branches
3654 }
3655
3656
3657 def main():
3658     if len(sys.argv[1:]) == 0:
3659         printUsage(commands.keys())
3660         sys.exit(2)
3661
3662     cmdName = sys.argv[1]
3663     try:
3664         klass = commands[cmdName]
3665         cmd = klass()
3666     except KeyError:
3667         print "unknown command %s" % cmdName
3668         print ""
3669         printUsage(commands.keys())
3670         sys.exit(2)
3671
3672     options = cmd.options
3673     cmd.gitdir = os.environ.get("GIT_DIR", None)
3674
3675     args = sys.argv[2:]
3676
3677     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3678     if cmd.needsGit:
3679         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3680
3681     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3682                                    options,
3683                                    description = cmd.description,
3684                                    formatter = HelpFormatter())
3685
3686     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3687     global verbose
3688     verbose = cmd.verbose
3689     if cmd.needsGit:
3690         if cmd.gitdir == None:
3691             cmd.gitdir = os.path.abspath(".git")
3692             if not isValidGitDir(cmd.gitdir):
3693                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3694                 if os.path.exists(cmd.gitdir):
3695                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3696                     if len(cdup) > 0:
3697                         chdir(cdup);
3698
3699         if not isValidGitDir(cmd.gitdir):
3700             if isValidGitDir(cmd.gitdir + "/.git"):
3701                 cmd.gitdir += "/.git"
3702             else:
3703                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3704
3705         os.environ["GIT_DIR"] = cmd.gitdir
3706
3707     if not cmd.run(args):
3708         parser.print_help()
3709         sys.exit(2)
3710
3711
3712 if __name__ == '__main__':
3713     main()