Merge branch 'da/mergetool-delete-delete-conflict'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28
29 try:
30     from subprocess import CalledProcessError
31 except ImportError:
32     # from python2.7:subprocess.py
33     # Exception classes used by this module.
34     class CalledProcessError(Exception):
35         """This exception is raised when a process run by check_call() returns
36         a non-zero exit status.  The exit status will be stored in the
37         returncode attribute."""
38         def __init__(self, returncode, cmd):
39             self.returncode = returncode
40             self.cmd = cmd
41         def __str__(self):
42             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
44 verbose = False
45
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
48
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
51
52 def p4_build_cmd(cmd):
53     """Build a suitable p4 command line.
54
55     This consolidates building and returning a p4 command line into one
56     location. It means that hooking into the environment, or other configuration
57     can be done more easily.
58     """
59     real_cmd = ["p4"]
60
61     user = gitConfig("git-p4.user")
62     if len(user) > 0:
63         real_cmd += ["-u",user]
64
65     password = gitConfig("git-p4.password")
66     if len(password) > 0:
67         real_cmd += ["-P", password]
68
69     port = gitConfig("git-p4.port")
70     if len(port) > 0:
71         real_cmd += ["-p", port]
72
73     host = gitConfig("git-p4.host")
74     if len(host) > 0:
75         real_cmd += ["-H", host]
76
77     client = gitConfig("git-p4.client")
78     if len(client) > 0:
79         real_cmd += ["-c", client]
80
81
82     if isinstance(cmd,basestring):
83         real_cmd = ' '.join(real_cmd) + ' ' + cmd
84     else:
85         real_cmd += cmd
86     return real_cmd
87
88 def chdir(path, is_client_path=False):
89     """Do chdir to the given path, and set the PWD environment
90        variable for use by P4.  It does not look at getcwd() output.
91        Since we're not using the shell, it is necessary to set the
92        PWD environment variable explicitly.
93
94        Normally, expand the path to force it to be absolute.  This
95        addresses the use of relative path names inside P4 settings,
96        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
97        as given; it looks for .p4config using PWD.
98
99        If is_client_path, the path was handed to us directly by p4,
100        and may be a symbolic link.  Do not call os.getcwd() in this
101        case, because it will cause p4 to think that PWD is not inside
102        the client path.
103        """
104
105     os.chdir(path)
106     if not is_client_path:
107         path = os.getcwd()
108     os.environ['PWD'] = path
109
110 def calcDiskFree():
111     """Return free space in bytes on the disk of the given dirname."""
112     if platform.system() == 'Windows':
113         free_bytes = ctypes.c_ulonglong(0)
114         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115         return free_bytes.value
116     else:
117         st = os.statvfs(os.getcwd())
118         return st.f_bavail * st.f_frsize
119
120 def die(msg):
121     if verbose:
122         raise Exception(msg)
123     else:
124         sys.stderr.write(msg + "\n")
125         sys.exit(1)
126
127 def write_pipe(c, stdin):
128     if verbose:
129         sys.stderr.write('Writing pipe: %s\n' % str(c))
130
131     expand = isinstance(c,basestring)
132     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133     pipe = p.stdin
134     val = pipe.write(stdin)
135     pipe.close()
136     if p.wait():
137         die('Command failed: %s' % str(c))
138
139     return val
140
141 def p4_write_pipe(c, stdin):
142     real_cmd = p4_build_cmd(c)
143     return write_pipe(real_cmd, stdin)
144
145 def read_pipe(c, ignore_error=False):
146     if verbose:
147         sys.stderr.write('Reading pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151     (out, err) = p.communicate()
152     if p.returncode != 0 and not ignore_error:
153         die('Command failed: %s\nError: %s' % (str(c), err))
154     return out
155
156 def p4_read_pipe(c, ignore_error=False):
157     real_cmd = p4_build_cmd(c)
158     return read_pipe(real_cmd, ignore_error)
159
160 def read_pipe_lines(c):
161     if verbose:
162         sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164     expand = isinstance(c, basestring)
165     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166     pipe = p.stdout
167     val = pipe.readlines()
168     if pipe.close() or p.wait():
169         die('Command failed: %s' % str(c))
170
171     return val
172
173 def p4_read_pipe_lines(c):
174     """Specifically invoke p4 on the command supplied. """
175     real_cmd = p4_build_cmd(c)
176     return read_pipe_lines(real_cmd)
177
178 def p4_has_command(cmd):
179     """Ask p4 for help on this command.  If it returns an error, the
180        command does not exist in this version of p4."""
181     real_cmd = p4_build_cmd(["help", cmd])
182     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183                                    stderr=subprocess.PIPE)
184     p.communicate()
185     return p.returncode == 0
186
187 def p4_has_move_command():
188     """See if the move command exists, that it supports -k, and that
189        it has not been administratively disabled.  The arguments
190        must be correct, but the filenames do not have to exist.  Use
191        ones with wildcards so even if they exist, it will fail."""
192
193     if not p4_has_command("move"):
194         return False
195     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197     (out, err) = p.communicate()
198     # return code will be 1 in either case
199     if err.find("Invalid option") >= 0:
200         return False
201     if err.find("disabled") >= 0:
202         return False
203     # assume it failed because @... was invalid changelist
204     return True
205
206 def system(cmd, ignore_error=False):
207     expand = isinstance(cmd,basestring)
208     if verbose:
209         sys.stderr.write("executing %s\n" % str(cmd))
210     retcode = subprocess.call(cmd, shell=expand)
211     if retcode and not ignore_error:
212         raise CalledProcessError(retcode, cmd)
213
214     return retcode
215
216 def p4_system(cmd):
217     """Specifically invoke p4 as the system command. """
218     real_cmd = p4_build_cmd(cmd)
219     expand = isinstance(real_cmd, basestring)
220     retcode = subprocess.call(real_cmd, shell=expand)
221     if retcode:
222         raise CalledProcessError(retcode, real_cmd)
223
224 _p4_version_string = None
225 def p4_version_string():
226     """Read the version string, showing just the last line, which
227        hopefully is the interesting version bit.
228
229        $ p4 -V
230        Perforce - The Fast Software Configuration Management System.
231        Copyright 1995-2011 Perforce Software.  All rights reserved.
232        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233     """
234     global _p4_version_string
235     if not _p4_version_string:
236         a = p4_read_pipe_lines(["-V"])
237         _p4_version_string = a[-1].rstrip()
238     return _p4_version_string
239
240 def p4_integrate(src, dest):
241     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
242
243 def p4_sync(f, *options):
244     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
245
246 def p4_add(f):
247     # forcibly add file names with wildcards
248     if wildcard_present(f):
249         p4_system(["add", "-f", f])
250     else:
251         p4_system(["add", f])
252
253 def p4_delete(f):
254     p4_system(["delete", wildcard_encode(f)])
255
256 def p4_edit(f, *options):
257     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
258
259 def p4_revert(f):
260     p4_system(["revert", wildcard_encode(f)])
261
262 def p4_reopen(type, f):
263     p4_system(["reopen", "-t", type, wildcard_encode(f)])
264
265 def p4_move(src, dest):
266     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
267
268 def p4_last_change():
269     results = p4CmdList(["changes", "-m", "1"])
270     return int(results[0]['change'])
271
272 def p4_describe(change):
273     """Make sure it returns a valid result by checking for
274        the presence of field "time".  Return a dict of the
275        results."""
276
277     ds = p4CmdList(["describe", "-s", str(change)])
278     if len(ds) != 1:
279         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280
281     d = ds[0]
282
283     if "p4ExitCode" in d:
284         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
285                                                       str(d)))
286     if "code" in d:
287         if d["code"] == "error":
288             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
289
290     if "time" not in d:
291         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
292
293     return d
294
295 #
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers.  See "p4 help filetypes"
298 # for a list and explanation.
299 #
300 def split_p4_type(p4type):
301
302     p4_filetypes_historical = {
303         "ctempobj": "binary+Sw",
304         "ctext": "text+C",
305         "cxtext": "text+Cx",
306         "ktext": "text+k",
307         "kxtext": "text+kx",
308         "ltext": "text+F",
309         "tempobj": "binary+FSw",
310         "ubinary": "binary+F",
311         "uresource": "resource+F",
312         "uxbinary": "binary+Fx",
313         "xbinary": "binary+x",
314         "xltext": "text+Fx",
315         "xtempobj": "binary+Swx",
316         "xtext": "text+x",
317         "xunicode": "unicode+x",
318         "xutf16": "utf16+x",
319     }
320     if p4type in p4_filetypes_historical:
321         p4type = p4_filetypes_historical[p4type]
322     mods = ""
323     s = p4type.split("+")
324     base = s[0]
325     mods = ""
326     if len(s) > 1:
327         mods = s[1]
328     return (base, mods)
329
330 #
331 # return the raw p4 type of a file (text, text+ko, etc)
332 #
333 def p4_type(f):
334     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
335     return results[0]['headType']
336
337 #
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
340 #
341 def p4_keywords_regexp_for_type(base, type_mods):
342     if base in ("text", "unicode", "binary"):
343         kwords = None
344         if "ko" in type_mods:
345             kwords = 'Id|Header'
346         elif "k" in type_mods:
347             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348         else:
349             return None
350         pattern = r"""
351             \$              # Starts with a dollar, followed by...
352             (%s)            # one of the keywords, followed by...
353             (:[^$\n]+)?     # possibly an old expansion, followed by...
354             \$              # another dollar
355             """ % kwords
356         return pattern
357     else:
358         return None
359
360 #
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
364 #
365 def p4_keywords_regexp_for_file(file):
366     if not os.path.exists(file):
367         return None
368     else:
369         (type_base, type_mods) = split_p4_type(p4_type(file))
370         return p4_keywords_regexp_for_type(type_base, type_mods)
371
372 def setP4ExecBit(file, mode):
373     # Reopens an already open file and changes the execute bit to match
374     # the execute bit setting in the passed in mode.
375
376     p4Type = "+x"
377
378     if not isModeExec(mode):
379         p4Type = getP4OpenedType(file)
380         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382         if p4Type[-1] == "+":
383             p4Type = p4Type[0:-1]
384
385     p4_reopen(p4Type, file)
386
387 def getP4OpenedType(file):
388     # Returns the perforce file type for the given file.
389
390     result = p4_read_pipe(["opened", wildcard_encode(file)])
391     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
392     if match:
393         return match.group(1)
394     else:
395         die("Could not determine file type for %s (result: '%s')" % (file, result))
396
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths):
399     labels = set()
400     if isinstance(depotPaths,basestring):
401         depotPaths = [depotPaths]
402
403     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
404         label = l['label']
405         labels.add(label)
406
407     return labels
408
409 # Return the set of all git tags
410 def getGitTags():
411     gitTags = set()
412     for line in read_pipe_lines(["git", "tag"]):
413         tag = line.strip()
414         gitTags.add(tag)
415     return gitTags
416
417 def diffTreePattern():
418     # This is a simple generator for the diff tree regex pattern. This could be
419     # a class variable if this and parseDiffTreeEntry were a part of a class.
420     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421     while True:
422         yield pattern
423
424 def parseDiffTreeEntry(entry):
425     """Parses a single diff tree entry into its component elements.
426
427     See git-diff-tree(1) manpage for details about the format of the diff
428     output. This method returns a dictionary with the following elements:
429
430     src_mode - The mode of the source file
431     dst_mode - The mode of the destination file
432     src_sha1 - The sha1 for the source file
433     dst_sha1 - The sha1 fr the destination file
434     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435     status_score - The score for the status (applicable for 'C' and 'R'
436                    statuses). This is None if there is no score.
437     src - The path for the source file.
438     dst - The path for the destination file. This is only present for
439           copy or renames. If it is not present, this is None.
440
441     If the pattern is not matched, None is returned."""
442
443     match = diffTreePattern().next().match(entry)
444     if match:
445         return {
446             'src_mode': match.group(1),
447             'dst_mode': match.group(2),
448             'src_sha1': match.group(3),
449             'dst_sha1': match.group(4),
450             'status': match.group(5),
451             'status_score': match.group(6),
452             'src': match.group(7),
453             'dst': match.group(10)
454         }
455     return None
456
457 def isModeExec(mode):
458     # Returns True if the given git mode represents an executable file,
459     # otherwise False.
460     return mode[-3:] == "755"
461
462 def isModeExecChanged(src_mode, dst_mode):
463     return isModeExec(src_mode) != isModeExec(dst_mode)
464
465 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
466
467     if isinstance(cmd,basestring):
468         cmd = "-G " + cmd
469         expand = True
470     else:
471         cmd = ["-G"] + cmd
472         expand = False
473
474     cmd = p4_build_cmd(cmd)
475     if verbose:
476         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
477
478     # Use a temporary file to avoid deadlocks without
479     # subprocess.communicate(), which would put another copy
480     # of stdout into memory.
481     stdin_file = None
482     if stdin is not None:
483         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
484         if isinstance(stdin,basestring):
485             stdin_file.write(stdin)
486         else:
487             for i in stdin:
488                 stdin_file.write(i + '\n')
489         stdin_file.flush()
490         stdin_file.seek(0)
491
492     p4 = subprocess.Popen(cmd,
493                           shell=expand,
494                           stdin=stdin_file,
495                           stdout=subprocess.PIPE)
496
497     result = []
498     try:
499         while True:
500             entry = marshal.load(p4.stdout)
501             if cb is not None:
502                 cb(entry)
503             else:
504                 result.append(entry)
505     except EOFError:
506         pass
507     exitCode = p4.wait()
508     if exitCode != 0:
509         entry = {}
510         entry["p4ExitCode"] = exitCode
511         result.append(entry)
512
513     return result
514
515 def p4Cmd(cmd):
516     list = p4CmdList(cmd)
517     result = {}
518     for entry in list:
519         result.update(entry)
520     return result;
521
522 def p4Where(depotPath):
523     if not depotPath.endswith("/"):
524         depotPath += "/"
525     depotPathLong = depotPath + "..."
526     outputList = p4CmdList(["where", depotPathLong])
527     output = None
528     for entry in outputList:
529         if "depotFile" in entry:
530             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531             # The base path always ends with "/...".
532             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
533                 output = entry
534                 break
535         elif "data" in entry:
536             data = entry.get("data")
537             space = data.find(" ")
538             if data[:space] == depotPath:
539                 output = entry
540                 break
541     if output == None:
542         return ""
543     if output["code"] == "error":
544         return ""
545     clientPath = ""
546     if "path" in output:
547         clientPath = output.get("path")
548     elif "data" in output:
549         data = output.get("data")
550         lastSpace = data.rfind(" ")
551         clientPath = data[lastSpace + 1:]
552
553     if clientPath.endswith("..."):
554         clientPath = clientPath[:-3]
555     return clientPath
556
557 def currentGitBranch():
558     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
559     if retcode != 0:
560         # on a detached head
561         return None
562     else:
563         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
564
565 def isValidGitDir(path):
566     if (os.path.exists(path + "/HEAD")
567         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
568         return True;
569     return False
570
571 def parseRevision(ref):
572     return read_pipe("git rev-parse %s" % ref).strip()
573
574 def branchExists(ref):
575     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
576                      ignore_error=True)
577     return len(rev) > 0
578
579 def extractLogMessageFromGitCommit(commit):
580     logMessage = ""
581
582     ## fixme: title is first line of commit, not 1st paragraph.
583     foundTitle = False
584     for log in read_pipe_lines("git cat-file commit %s" % commit):
585        if not foundTitle:
586            if len(log) == 1:
587                foundTitle = True
588            continue
589
590        logMessage += log
591     return logMessage
592
593 def extractSettingsGitLog(log):
594     values = {}
595     for line in log.split("\n"):
596         line = line.strip()
597         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
598         if not m:
599             continue
600
601         assignments = m.group(1).split (':')
602         for a in assignments:
603             vals = a.split ('=')
604             key = vals[0].strip()
605             val = ('='.join (vals[1:])).strip()
606             if val.endswith ('\"') and val.startswith('"'):
607                 val = val[1:-1]
608
609             values[key] = val
610
611     paths = values.get("depot-paths")
612     if not paths:
613         paths = values.get("depot-path")
614     if paths:
615         values['depot-paths'] = paths.split(',')
616     return values
617
618 def gitBranchExists(branch):
619     proc = subprocess.Popen(["git", "rev-parse", branch],
620                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
621     return proc.wait() == 0;
622
623 _gitConfig = {}
624
625 def gitConfig(key, typeSpecifier=None):
626     if not _gitConfig.has_key(key):
627         cmd = [ "git", "config" ]
628         if typeSpecifier:
629             cmd += [ typeSpecifier ]
630         cmd += [ key ]
631         s = read_pipe(cmd, ignore_error=True)
632         _gitConfig[key] = s.strip()
633     return _gitConfig[key]
634
635 def gitConfigBool(key):
636     """Return a bool, using git config --bool.  It is True only if the
637        variable is set to true, and False if set to false or not present
638        in the config."""
639
640     if not _gitConfig.has_key(key):
641         _gitConfig[key] = gitConfig(key, '--bool') == "true"
642     return _gitConfig[key]
643
644 def gitConfigInt(key):
645     if not _gitConfig.has_key(key):
646         cmd = [ "git", "config", "--int", key ]
647         s = read_pipe(cmd, ignore_error=True)
648         v = s.strip()
649         try:
650             _gitConfig[key] = int(gitConfig(key, '--int'))
651         except ValueError:
652             _gitConfig[key] = None
653     return _gitConfig[key]
654
655 def gitConfigList(key):
656     if not _gitConfig.has_key(key):
657         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658         _gitConfig[key] = s.strip().split(os.linesep)
659         if _gitConfig[key] == ['']:
660             _gitConfig[key] = []
661     return _gitConfig[key]
662
663 def p4BranchesInGit(branchesAreInRemotes=True):
664     """Find all the branches whose names start with "p4/", looking
665        in remotes or heads as specified by the argument.  Return
666        a dictionary of { branch: revision } for each one found.
667        The branch names are the short names, without any
668        "p4/" prefix."""
669
670     branches = {}
671
672     cmdline = "git rev-parse --symbolic "
673     if branchesAreInRemotes:
674         cmdline += "--remotes"
675     else:
676         cmdline += "--branches"
677
678     for line in read_pipe_lines(cmdline):
679         line = line.strip()
680
681         # only import to p4/
682         if not line.startswith('p4/'):
683             continue
684         # special symbolic ref to p4/master
685         if line == "p4/HEAD":
686             continue
687
688         # strip off p4/ prefix
689         branch = line[len("p4/"):]
690
691         branches[branch] = parseRevision(line)
692
693     return branches
694
695 def branch_exists(branch):
696     """Make sure that the given ref name really exists."""
697
698     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700     out, _ = p.communicate()
701     if p.returncode:
702         return False
703     # expect exactly one line of output: the branch name
704     return out.rstrip() == branch
705
706 def findUpstreamBranchPoint(head = "HEAD"):
707     branches = p4BranchesInGit()
708     # map from depot-path to branch name
709     branchByDepotPath = {}
710     for branch in branches.keys():
711         tip = branches[branch]
712         log = extractLogMessageFromGitCommit(tip)
713         settings = extractSettingsGitLog(log)
714         if settings.has_key("depot-paths"):
715             paths = ",".join(settings["depot-paths"])
716             branchByDepotPath[paths] = "remotes/p4/" + branch
717
718     settings = None
719     parent = 0
720     while parent < 65535:
721         commit = head + "~%s" % parent
722         log = extractLogMessageFromGitCommit(commit)
723         settings = extractSettingsGitLog(log)
724         if settings.has_key("depot-paths"):
725             paths = ",".join(settings["depot-paths"])
726             if branchByDepotPath.has_key(paths):
727                 return [branchByDepotPath[paths], settings]
728
729         parent = parent + 1
730
731     return ["", settings]
732
733 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
734     if not silent:
735         print ("Creating/updating branch(es) in %s based on origin branch(es)"
736                % localRefPrefix)
737
738     originPrefix = "origin/p4/"
739
740     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
741         line = line.strip()
742         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
743             continue
744
745         headName = line[len(originPrefix):]
746         remoteHead = localRefPrefix + headName
747         originHead = line
748
749         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750         if (not original.has_key('depot-paths')
751             or not original.has_key('change')):
752             continue
753
754         update = False
755         if not gitBranchExists(remoteHead):
756             if verbose:
757                 print "creating %s" % remoteHead
758             update = True
759         else:
760             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761             if settings.has_key('change') > 0:
762                 if settings['depot-paths'] == original['depot-paths']:
763                     originP4Change = int(original['change'])
764                     p4Change = int(settings['change'])
765                     if originP4Change > p4Change:
766                         print ("%s (%s) is newer than %s (%s). "
767                                "Updating p4 branch from origin."
768                                % (originHead, originP4Change,
769                                   remoteHead, p4Change))
770                         update = True
771                 else:
772                     print ("Ignoring: %s was imported from %s while "
773                            "%s was imported from %s"
774                            % (originHead, ','.join(original['depot-paths']),
775                               remoteHead, ','.join(settings['depot-paths'])))
776
777         if update:
778             system("git update-ref %s %s" % (remoteHead, originHead))
779
780 def originP4BranchesExist():
781         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
782
783
784 def p4ParseNumericChangeRange(parts):
785     changeStart = int(parts[0][1:])
786     if parts[1] == '#head':
787         changeEnd = p4_last_change()
788     else:
789         changeEnd = int(parts[1])
790
791     return (changeStart, changeEnd)
792
793 def chooseBlockSize(blockSize):
794     if blockSize:
795         return blockSize
796     else:
797         return defaultBlockSize
798
799 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
800     assert depotPaths
801
802     # Parse the change range into start and end. Try to find integer
803     # revision ranges as these can be broken up into blocks to avoid
804     # hitting server-side limits (maxrows, maxscanresults). But if
805     # that doesn't work, fall back to using the raw revision specifier
806     # strings, without using block mode.
807
808     if changeRange is None or changeRange == '':
809         changeStart = 1
810         changeEnd = p4_last_change()
811         block_size = chooseBlockSize(requestedBlockSize)
812     else:
813         parts = changeRange.split(',')
814         assert len(parts) == 2
815         try:
816             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817             block_size = chooseBlockSize(requestedBlockSize)
818         except:
819             changeStart = parts[0][1:]
820             changeEnd = parts[1]
821             if requestedBlockSize:
822                 die("cannot use --changes-block-size with non-numeric revisions")
823             block_size = None
824
825     changes = []
826
827     # Retrieve changes a block at a time, to prevent running
828     # into a MaxResults/MaxScanRows error from the server.
829
830     while True:
831         cmd = ['changes']
832
833         if block_size:
834             end = min(changeEnd, changeStart + block_size)
835             revisionRange = "%d,%d" % (changeStart, end)
836         else:
837             revisionRange = "%s,%s" % (changeStart, changeEnd)
838
839         for p in depotPaths:
840             cmd += ["%s...@%s" % (p, revisionRange)]
841
842         # Insert changes in chronological order
843         for line in reversed(p4_read_pipe_lines(cmd)):
844             changes.append(int(line.split(" ")[1]))
845
846         if not block_size:
847             break
848
849         if end >= changeEnd:
850             break
851
852         changeStart = end + 1
853
854     changes = sorted(changes)
855     return changes
856
857 def p4PathStartsWith(path, prefix):
858     # This method tries to remedy a potential mixed-case issue:
859     #
860     # If UserA adds  //depot/DirA/file1
861     # and UserB adds //depot/dira/file2
862     #
863     # we may or may not have a problem. If you have core.ignorecase=true,
864     # we treat DirA and dira as the same directory
865     if gitConfigBool("core.ignorecase"):
866         return path.lower().startswith(prefix.lower())
867     return path.startswith(prefix)
868
869 def getClientSpec():
870     """Look at the p4 client spec, create a View() object that contains
871        all the mappings, and return it."""
872
873     specList = p4CmdList("client -o")
874     if len(specList) != 1:
875         die('Output from "client -o" is %d lines, expecting 1' %
876             len(specList))
877
878     # dictionary of all client parameters
879     entry = specList[0]
880
881     # the //client/ name
882     client_name = entry["Client"]
883
884     # just the keys that start with "View"
885     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
886
887     # hold this new View
888     view = View(client_name)
889
890     # append the lines, in order, to the view
891     for view_num in range(len(view_keys)):
892         k = "View%d" % view_num
893         if k not in view_keys:
894             die("Expected view key %s missing" % k)
895         view.append(entry[k])
896
897     return view
898
899 def getClientRoot():
900     """Grab the client directory."""
901
902     output = p4CmdList("client -o")
903     if len(output) != 1:
904         die('Output from "client -o" is %d lines, expecting 1' % len(output))
905
906     entry = output[0]
907     if "Root" not in entry:
908         die('Client has no "Root"')
909
910     return entry["Root"]
911
912 #
913 # P4 wildcards are not allowed in filenames.  P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
916 #
917 def wildcard_decode(path):
918     # Search for and fix just these four characters.  Do % last so
919     # that fixing it does not inadvertently create new %-escapes.
920     # Cannot have * in a filename in windows; untested as to
921     # what p4 would do in such a case.
922     if not platform.system() == "Windows":
923         path = path.replace("%2A", "*")
924     path = path.replace("%23", "#") \
925                .replace("%40", "@") \
926                .replace("%25", "%")
927     return path
928
929 def wildcard_encode(path):
930     # do % first to avoid double-encoding the %s introduced here
931     path = path.replace("%", "%25") \
932                .replace("*", "%2A") \
933                .replace("#", "%23") \
934                .replace("@", "%40")
935     return path
936
937 def wildcard_present(path):
938     m = re.search("[*#@%]", path)
939     return m is not None
940
941 class LargeFileSystem(object):
942     """Base class for large file system support."""
943
944     def __init__(self, writeToGitStream):
945         self.largeFiles = set()
946         self.writeToGitStream = writeToGitStream
947
948     def generatePointer(self, cloneDestination, contentFile):
949         """Return the content of a pointer file that is stored in Git instead of
950            the actual content."""
951         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
952
953     def pushFile(self, localLargeFile):
954         """Push the actual content which is not stored in the Git repository to
955            a server."""
956         assert False, "Method 'pushFile' required in " + self.__class__.__name__
957
958     def hasLargeFileExtension(self, relPath):
959         return reduce(
960             lambda a, b: a or b,
961             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
962             False
963         )
964
965     def generateTempFile(self, contents):
966         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
967         for d in contents:
968             contentFile.write(d)
969         contentFile.close()
970         return contentFile.name
971
972     def exceedsLargeFileThreshold(self, relPath, contents):
973         if gitConfigInt('git-p4.largeFileThreshold'):
974             contentsSize = sum(len(d) for d in contents)
975             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
976                 return True
977         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978             contentsSize = sum(len(d) for d in contents)
979             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
980                 return False
981             contentTempFile = self.generateTempFile(contents)
982             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
985             zf.close()
986             compressedContentsSize = zf.infolist()[0].compress_size
987             os.remove(contentTempFile)
988             os.remove(compressedContentFile.name)
989             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
990                 return True
991         return False
992
993     def addLargeFile(self, relPath):
994         self.largeFiles.add(relPath)
995
996     def removeLargeFile(self, relPath):
997         self.largeFiles.remove(relPath)
998
999     def isLargeFile(self, relPath):
1000         return relPath in self.largeFiles
1001
1002     def processContent(self, git_mode, relPath, contents):
1003         """Processes the content of git fast import. This method decides if a
1004            file is stored in the large file system and handles all necessary
1005            steps."""
1006         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007             contentTempFile = self.generateTempFile(contents)
1008             (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1009
1010             # Move temp file to final location in large file system
1011             largeFileDir = os.path.dirname(localLargeFile)
1012             if not os.path.isdir(largeFileDir):
1013                 os.makedirs(largeFileDir)
1014             shutil.move(contentTempFile, localLargeFile)
1015             self.addLargeFile(relPath)
1016             if gitConfigBool('git-p4.largeFilePush'):
1017                 self.pushFile(localLargeFile)
1018             if verbose:
1019                 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020         return (git_mode, contents)
1021
1022 class MockLFS(LargeFileSystem):
1023     """Mock large file system for testing."""
1024
1025     def generatePointer(self, contentFile):
1026         """The pointer content is the original content prefixed with "pointer-".
1027            The local filename of the large file storage is derived from the file content.
1028            """
1029         with open(contentFile, 'r') as f:
1030             content = next(f)
1031             gitMode = '100644'
1032             pointerContents = 'pointer-' + content
1033             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034             return (gitMode, pointerContents, localLargeFile)
1035
1036     def pushFile(self, localLargeFile):
1037         """The remote filename of the large file storage is the same as the local
1038            one but in a different directory.
1039            """
1040         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041         if not os.path.exists(remotePath):
1042             os.makedirs(remotePath)
1043         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1044
1045 class GitLFS(LargeFileSystem):
1046     """Git LFS as backend for the git-p4 large file system.
1047        See https://git-lfs.github.com/ for details."""
1048
1049     def __init__(self, *args):
1050         LargeFileSystem.__init__(self, *args)
1051         self.baseGitAttributes = []
1052
1053     def generatePointer(self, contentFile):
1054         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055            mode and content which is stored in the Git repository instead of
1056            the actual content. Return also the new location of the actual
1057            content.
1058            """
1059         pointerProcess = subprocess.Popen(
1060             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061             stdout=subprocess.PIPE
1062         )
1063         pointerFile = pointerProcess.stdout.read()
1064         if pointerProcess.wait():
1065             os.remove(contentFile)
1066             die('git-lfs pointer command failed. Did you install the extension?')
1067         pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
1068         oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
1069         localLargeFile = os.path.join(
1070             os.getcwd(),
1071             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1072             oid,
1073         )
1074         # LFS Spec states that pointer files should not have the executable bit set.
1075         gitMode = '100644'
1076         return (gitMode, pointerContents, localLargeFile)
1077
1078     def pushFile(self, localLargeFile):
1079         uploadProcess = subprocess.Popen(
1080             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1081         )
1082         if uploadProcess.wait():
1083             die('git-lfs push command failed. Did you define a remote?')
1084
1085     def generateGitAttributes(self):
1086         return (
1087             self.baseGitAttributes +
1088             [
1089                 '\n',
1090                 '#\n',
1091                 '# Git LFS (see https://git-lfs.github.com/)\n',
1092                 '#\n',
1093             ] +
1094             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1095                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1096             ] +
1097             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1098                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1099             ]
1100         )
1101
1102     def addLargeFile(self, relPath):
1103         LargeFileSystem.addLargeFile(self, relPath)
1104         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1105
1106     def removeLargeFile(self, relPath):
1107         LargeFileSystem.removeLargeFile(self, relPath)
1108         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1109
1110     def processContent(self, git_mode, relPath, contents):
1111         if relPath == '.gitattributes':
1112             self.baseGitAttributes = contents
1113             return (git_mode, self.generateGitAttributes())
1114         else:
1115             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1116
1117 class Command:
1118     def __init__(self):
1119         self.usage = "usage: %prog [options]"
1120         self.needsGit = True
1121         self.verbose = False
1122
1123 class P4UserMap:
1124     def __init__(self):
1125         self.userMapFromPerforceServer = False
1126         self.myP4UserId = None
1127
1128     def p4UserId(self):
1129         if self.myP4UserId:
1130             return self.myP4UserId
1131
1132         results = p4CmdList("user -o")
1133         for r in results:
1134             if r.has_key('User'):
1135                 self.myP4UserId = r['User']
1136                 return r['User']
1137         die("Could not find your p4 user id")
1138
1139     def p4UserIsMe(self, p4User):
1140         # return True if the given p4 user is actually me
1141         me = self.p4UserId()
1142         if not p4User or p4User != me:
1143             return False
1144         else:
1145             return True
1146
1147     def getUserCacheFilename(self):
1148         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1149         return home + "/.gitp4-usercache.txt"
1150
1151     def getUserMapFromPerforceServer(self):
1152         if self.userMapFromPerforceServer:
1153             return
1154         self.users = {}
1155         self.emails = {}
1156
1157         for output in p4CmdList("users"):
1158             if not output.has_key("User"):
1159                 continue
1160             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1161             self.emails[output["Email"]] = output["User"]
1162
1163
1164         s = ''
1165         for (key, val) in self.users.items():
1166             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1167
1168         open(self.getUserCacheFilename(), "wb").write(s)
1169         self.userMapFromPerforceServer = True
1170
1171     def loadUserMapFromCache(self):
1172         self.users = {}
1173         self.userMapFromPerforceServer = False
1174         try:
1175             cache = open(self.getUserCacheFilename(), "rb")
1176             lines = cache.readlines()
1177             cache.close()
1178             for line in lines:
1179                 entry = line.strip().split("\t")
1180                 self.users[entry[0]] = entry[1]
1181         except IOError:
1182             self.getUserMapFromPerforceServer()
1183
1184 class P4Debug(Command):
1185     def __init__(self):
1186         Command.__init__(self)
1187         self.options = []
1188         self.description = "A tool to debug the output of p4 -G."
1189         self.needsGit = False
1190
1191     def run(self, args):
1192         j = 0
1193         for output in p4CmdList(args):
1194             print 'Element: %d' % j
1195             j += 1
1196             print output
1197         return True
1198
1199 class P4RollBack(Command):
1200     def __init__(self):
1201         Command.__init__(self)
1202         self.options = [
1203             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1204         ]
1205         self.description = "A tool to debug the multi-branch import. Don't use :)"
1206         self.rollbackLocalBranches = False
1207
1208     def run(self, args):
1209         if len(args) != 1:
1210             return False
1211         maxChange = int(args[0])
1212
1213         if "p4ExitCode" in p4Cmd("changes -m 1"):
1214             die("Problems executing p4");
1215
1216         if self.rollbackLocalBranches:
1217             refPrefix = "refs/heads/"
1218             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1219         else:
1220             refPrefix = "refs/remotes/"
1221             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1222
1223         for line in lines:
1224             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1225                 line = line.strip()
1226                 ref = refPrefix + line
1227                 log = extractLogMessageFromGitCommit(ref)
1228                 settings = extractSettingsGitLog(log)
1229
1230                 depotPaths = settings['depot-paths']
1231                 change = settings['change']
1232
1233                 changed = False
1234
1235                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1236                                                            for p in depotPaths]))) == 0:
1237                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1238                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1239                     continue
1240
1241                 while change and int(change) > maxChange:
1242                     changed = True
1243                     if self.verbose:
1244                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1245                     system("git update-ref %s \"%s^\"" % (ref, ref))
1246                     log = extractLogMessageFromGitCommit(ref)
1247                     settings =  extractSettingsGitLog(log)
1248
1249
1250                     depotPaths = settings['depot-paths']
1251                     change = settings['change']
1252
1253                 if changed:
1254                     print "%s rewound to %s" % (ref, change)
1255
1256         return True
1257
1258 class P4Submit(Command, P4UserMap):
1259
1260     conflict_behavior_choices = ("ask", "skip", "quit")
1261
1262     def __init__(self):
1263         Command.__init__(self)
1264         P4UserMap.__init__(self)
1265         self.options = [
1266                 optparse.make_option("--origin", dest="origin"),
1267                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1268                 # preserve the user, requires relevant p4 permissions
1269                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1270                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1271                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1272                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1273                 optparse.make_option("--conflict", dest="conflict_behavior",
1274                                      choices=self.conflict_behavior_choices),
1275                 optparse.make_option("--branch", dest="branch"),
1276         ]
1277         self.description = "Submit changes from git to the perforce depot."
1278         self.usage += " [name of git branch to submit into perforce depot]"
1279         self.origin = ""
1280         self.detectRenames = False
1281         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1282         self.dry_run = False
1283         self.prepare_p4_only = False
1284         self.conflict_behavior = None
1285         self.isWindows = (platform.system() == "Windows")
1286         self.exportLabels = False
1287         self.p4HasMoveCommand = p4_has_move_command()
1288         self.branch = None
1289
1290         if gitConfig('git-p4.largeFileSystem'):
1291             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1292
1293     def check(self):
1294         if len(p4CmdList("opened ...")) > 0:
1295             die("You have files opened with perforce! Close them before starting the sync.")
1296
1297     def separate_jobs_from_description(self, message):
1298         """Extract and return a possible Jobs field in the commit
1299            message.  It goes into a separate section in the p4 change
1300            specification.
1301
1302            A jobs line starts with "Jobs:" and looks like a new field
1303            in a form.  Values are white-space separated on the same
1304            line or on following lines that start with a tab.
1305
1306            This does not parse and extract the full git commit message
1307            like a p4 form.  It just sees the Jobs: line as a marker
1308            to pass everything from then on directly into the p4 form,
1309            but outside the description section.
1310
1311            Return a tuple (stripped log message, jobs string)."""
1312
1313         m = re.search(r'^Jobs:', message, re.MULTILINE)
1314         if m is None:
1315             return (message, None)
1316
1317         jobtext = message[m.start():]
1318         stripped_message = message[:m.start()].rstrip()
1319         return (stripped_message, jobtext)
1320
1321     def prepareLogMessage(self, template, message, jobs):
1322         """Edits the template returned from "p4 change -o" to insert
1323            the message in the Description field, and the jobs text in
1324            the Jobs field."""
1325         result = ""
1326
1327         inDescriptionSection = False
1328
1329         for line in template.split("\n"):
1330             if line.startswith("#"):
1331                 result += line + "\n"
1332                 continue
1333
1334             if inDescriptionSection:
1335                 if line.startswith("Files:") or line.startswith("Jobs:"):
1336                     inDescriptionSection = False
1337                     # insert Jobs section
1338                     if jobs:
1339                         result += jobs + "\n"
1340                 else:
1341                     continue
1342             else:
1343                 if line.startswith("Description:"):
1344                     inDescriptionSection = True
1345                     line += "\n"
1346                     for messageLine in message.split("\n"):
1347                         line += "\t" + messageLine + "\n"
1348
1349             result += line + "\n"
1350
1351         return result
1352
1353     def patchRCSKeywords(self, file, pattern):
1354         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1355         (handle, outFileName) = tempfile.mkstemp(dir='.')
1356         try:
1357             outFile = os.fdopen(handle, "w+")
1358             inFile = open(file, "r")
1359             regexp = re.compile(pattern, re.VERBOSE)
1360             for line in inFile.readlines():
1361                 line = regexp.sub(r'$\1$', line)
1362                 outFile.write(line)
1363             inFile.close()
1364             outFile.close()
1365             # Forcibly overwrite the original file
1366             os.unlink(file)
1367             shutil.move(outFileName, file)
1368         except:
1369             # cleanup our temporary file
1370             os.unlink(outFileName)
1371             print "Failed to strip RCS keywords in %s" % file
1372             raise
1373
1374         print "Patched up RCS keywords in %s" % file
1375
1376     def p4UserForCommit(self,id):
1377         # Return the tuple (perforce user,git email) for a given git commit id
1378         self.getUserMapFromPerforceServer()
1379         gitEmail = read_pipe(["git", "log", "--max-count=1",
1380                               "--format=%ae", id])
1381         gitEmail = gitEmail.strip()
1382         if not self.emails.has_key(gitEmail):
1383             return (None,gitEmail)
1384         else:
1385             return (self.emails[gitEmail],gitEmail)
1386
1387     def checkValidP4Users(self,commits):
1388         # check if any git authors cannot be mapped to p4 users
1389         for id in commits:
1390             (user,email) = self.p4UserForCommit(id)
1391             if not user:
1392                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1393                 if gitConfigBool("git-p4.allowMissingP4Users"):
1394                     print "%s" % msg
1395                 else:
1396                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1397
1398     def lastP4Changelist(self):
1399         # Get back the last changelist number submitted in this client spec. This
1400         # then gets used to patch up the username in the change. If the same
1401         # client spec is being used by multiple processes then this might go
1402         # wrong.
1403         results = p4CmdList("client -o")        # find the current client
1404         client = None
1405         for r in results:
1406             if r.has_key('Client'):
1407                 client = r['Client']
1408                 break
1409         if not client:
1410             die("could not get client spec")
1411         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1412         for r in results:
1413             if r.has_key('change'):
1414                 return r['change']
1415         die("Could not get changelist number for last submit - cannot patch up user details")
1416
1417     def modifyChangelistUser(self, changelist, newUser):
1418         # fixup the user field of a changelist after it has been submitted.
1419         changes = p4CmdList("change -o %s" % changelist)
1420         if len(changes) != 1:
1421             die("Bad output from p4 change modifying %s to user %s" %
1422                 (changelist, newUser))
1423
1424         c = changes[0]
1425         if c['User'] == newUser: return   # nothing to do
1426         c['User'] = newUser
1427         input = marshal.dumps(c)
1428
1429         result = p4CmdList("change -f -i", stdin=input)
1430         for r in result:
1431             if r.has_key('code'):
1432                 if r['code'] == 'error':
1433                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1434             if r.has_key('data'):
1435                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1436                 return
1437         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1438
1439     def canChangeChangelists(self):
1440         # check to see if we have p4 admin or super-user permissions, either of
1441         # which are required to modify changelists.
1442         results = p4CmdList(["protects", self.depotPath])
1443         for r in results:
1444             if r.has_key('perm'):
1445                 if r['perm'] == 'admin':
1446                     return 1
1447                 if r['perm'] == 'super':
1448                     return 1
1449         return 0
1450
1451     def prepareSubmitTemplate(self):
1452         """Run "p4 change -o" to grab a change specification template.
1453            This does not use "p4 -G", as it is nice to keep the submission
1454            template in original order, since a human might edit it.
1455
1456            Remove lines in the Files section that show changes to files
1457            outside the depot path we're committing into."""
1458
1459         [upstream, settings] = findUpstreamBranchPoint()
1460
1461         template = ""
1462         inFilesSection = False
1463         for line in p4_read_pipe_lines(['change', '-o']):
1464             if line.endswith("\r\n"):
1465                 line = line[:-2] + "\n"
1466             if inFilesSection:
1467                 if line.startswith("\t"):
1468                     # path starts and ends with a tab
1469                     path = line[1:]
1470                     lastTab = path.rfind("\t")
1471                     if lastTab != -1:
1472                         path = path[:lastTab]
1473                         if settings.has_key('depot-paths'):
1474                             if not [p for p in settings['depot-paths']
1475                                     if p4PathStartsWith(path, p)]:
1476                                 continue
1477                         else:
1478                             if not p4PathStartsWith(path, self.depotPath):
1479                                 continue
1480                 else:
1481                     inFilesSection = False
1482             else:
1483                 if line.startswith("Files:"):
1484                     inFilesSection = True
1485
1486             template += line
1487
1488         return template
1489
1490     def edit_template(self, template_file):
1491         """Invoke the editor to let the user change the submission
1492            message.  Return true if okay to continue with the submit."""
1493
1494         # if configured to skip the editing part, just submit
1495         if gitConfigBool("git-p4.skipSubmitEdit"):
1496             return True
1497
1498         # look at the modification time, to check later if the user saved
1499         # the file
1500         mtime = os.stat(template_file).st_mtime
1501
1502         # invoke the editor
1503         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1504             editor = os.environ.get("P4EDITOR")
1505         else:
1506             editor = read_pipe("git var GIT_EDITOR").strip()
1507         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1508
1509         # If the file was not saved, prompt to see if this patch should
1510         # be skipped.  But skip this verification step if configured so.
1511         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1512             return True
1513
1514         # modification time updated means user saved the file
1515         if os.stat(template_file).st_mtime > mtime:
1516             return True
1517
1518         while True:
1519             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1520             if response == 'y':
1521                 return True
1522             if response == 'n':
1523                 return False
1524
1525     def get_diff_description(self, editedFiles, filesToAdd):
1526         # diff
1527         if os.environ.has_key("P4DIFF"):
1528             del(os.environ["P4DIFF"])
1529         diff = ""
1530         for editedFile in editedFiles:
1531             diff += p4_read_pipe(['diff', '-du',
1532                                   wildcard_encode(editedFile)])
1533
1534         # new file diff
1535         newdiff = ""
1536         for newFile in filesToAdd:
1537             newdiff += "==== new file ====\n"
1538             newdiff += "--- /dev/null\n"
1539             newdiff += "+++ %s\n" % newFile
1540             f = open(newFile, "r")
1541             for line in f.readlines():
1542                 newdiff += "+" + line
1543             f.close()
1544
1545         return (diff + newdiff).replace('\r\n', '\n')
1546
1547     def applyCommit(self, id):
1548         """Apply one commit, return True if it succeeded."""
1549
1550         print "Applying", read_pipe(["git", "show", "-s",
1551                                      "--format=format:%h %s", id])
1552
1553         (p4User, gitEmail) = self.p4UserForCommit(id)
1554
1555         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1556         filesToAdd = set()
1557         filesToChangeType = set()
1558         filesToDelete = set()
1559         editedFiles = set()
1560         pureRenameCopy = set()
1561         filesToChangeExecBit = {}
1562
1563         for line in diff:
1564             diff = parseDiffTreeEntry(line)
1565             modifier = diff['status']
1566             path = diff['src']
1567             if modifier == "M":
1568                 p4_edit(path)
1569                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1570                     filesToChangeExecBit[path] = diff['dst_mode']
1571                 editedFiles.add(path)
1572             elif modifier == "A":
1573                 filesToAdd.add(path)
1574                 filesToChangeExecBit[path] = diff['dst_mode']
1575                 if path in filesToDelete:
1576                     filesToDelete.remove(path)
1577             elif modifier == "D":
1578                 filesToDelete.add(path)
1579                 if path in filesToAdd:
1580                     filesToAdd.remove(path)
1581             elif modifier == "C":
1582                 src, dest = diff['src'], diff['dst']
1583                 p4_integrate(src, dest)
1584                 pureRenameCopy.add(dest)
1585                 if diff['src_sha1'] != diff['dst_sha1']:
1586                     p4_edit(dest)
1587                     pureRenameCopy.discard(dest)
1588                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1589                     p4_edit(dest)
1590                     pureRenameCopy.discard(dest)
1591                     filesToChangeExecBit[dest] = diff['dst_mode']
1592                 if self.isWindows:
1593                     # turn off read-only attribute
1594                     os.chmod(dest, stat.S_IWRITE)
1595                 os.unlink(dest)
1596                 editedFiles.add(dest)
1597             elif modifier == "R":
1598                 src, dest = diff['src'], diff['dst']
1599                 if self.p4HasMoveCommand:
1600                     p4_edit(src)        # src must be open before move
1601                     p4_move(src, dest)  # opens for (move/delete, move/add)
1602                 else:
1603                     p4_integrate(src, dest)
1604                     if diff['src_sha1'] != diff['dst_sha1']:
1605                         p4_edit(dest)
1606                     else:
1607                         pureRenameCopy.add(dest)
1608                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1609                     if not self.p4HasMoveCommand:
1610                         p4_edit(dest)   # with move: already open, writable
1611                     filesToChangeExecBit[dest] = diff['dst_mode']
1612                 if not self.p4HasMoveCommand:
1613                     if self.isWindows:
1614                         os.chmod(dest, stat.S_IWRITE)
1615                     os.unlink(dest)
1616                     filesToDelete.add(src)
1617                 editedFiles.add(dest)
1618             elif modifier == "T":
1619                 filesToChangeType.add(path)
1620             else:
1621                 die("unknown modifier %s for %s" % (modifier, path))
1622
1623         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1624         patchcmd = diffcmd + " | git apply "
1625         tryPatchCmd = patchcmd + "--check -"
1626         applyPatchCmd = patchcmd + "--check --apply -"
1627         patch_succeeded = True
1628
1629         if os.system(tryPatchCmd) != 0:
1630             fixed_rcs_keywords = False
1631             patch_succeeded = False
1632             print "Unfortunately applying the change failed!"
1633
1634             # Patch failed, maybe it's just RCS keyword woes. Look through
1635             # the patch to see if that's possible.
1636             if gitConfigBool("git-p4.attemptRCSCleanup"):
1637                 file = None
1638                 pattern = None
1639                 kwfiles = {}
1640                 for file in editedFiles | filesToDelete:
1641                     # did this file's delta contain RCS keywords?
1642                     pattern = p4_keywords_regexp_for_file(file)
1643
1644                     if pattern:
1645                         # this file is a possibility...look for RCS keywords.
1646                         regexp = re.compile(pattern, re.VERBOSE)
1647                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1648                             if regexp.search(line):
1649                                 if verbose:
1650                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1651                                 kwfiles[file] = pattern
1652                                 break
1653
1654                 for file in kwfiles:
1655                     if verbose:
1656                         print "zapping %s with %s" % (line,pattern)
1657                     # File is being deleted, so not open in p4.  Must
1658                     # disable the read-only bit on windows.
1659                     if self.isWindows and file not in editedFiles:
1660                         os.chmod(file, stat.S_IWRITE)
1661                     self.patchRCSKeywords(file, kwfiles[file])
1662                     fixed_rcs_keywords = True
1663
1664             if fixed_rcs_keywords:
1665                 print "Retrying the patch with RCS keywords cleaned up"
1666                 if os.system(tryPatchCmd) == 0:
1667                     patch_succeeded = True
1668
1669         if not patch_succeeded:
1670             for f in editedFiles:
1671                 p4_revert(f)
1672             return False
1673
1674         #
1675         # Apply the patch for real, and do add/delete/+x handling.
1676         #
1677         system(applyPatchCmd)
1678
1679         for f in filesToChangeType:
1680             p4_edit(f, "-t", "auto")
1681         for f in filesToAdd:
1682             p4_add(f)
1683         for f in filesToDelete:
1684             p4_revert(f)
1685             p4_delete(f)
1686
1687         # Set/clear executable bits
1688         for f in filesToChangeExecBit.keys():
1689             mode = filesToChangeExecBit[f]
1690             setP4ExecBit(f, mode)
1691
1692         #
1693         # Build p4 change description, starting with the contents
1694         # of the git commit message.
1695         #
1696         logMessage = extractLogMessageFromGitCommit(id)
1697         logMessage = logMessage.strip()
1698         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1699
1700         template = self.prepareSubmitTemplate()
1701         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1702
1703         if self.preserveUser:
1704            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1705
1706         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1707             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1708             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1709             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1710
1711         separatorLine = "######## everything below this line is just the diff #######\n"
1712         if not self.prepare_p4_only:
1713             submitTemplate += separatorLine
1714             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1715
1716         (handle, fileName) = tempfile.mkstemp()
1717         tmpFile = os.fdopen(handle, "w+b")
1718         if self.isWindows:
1719             submitTemplate = submitTemplate.replace("\n", "\r\n")
1720         tmpFile.write(submitTemplate)
1721         tmpFile.close()
1722
1723         if self.prepare_p4_only:
1724             #
1725             # Leave the p4 tree prepared, and the submit template around
1726             # and let the user decide what to do next
1727             #
1728             print
1729             print "P4 workspace prepared for submission."
1730             print "To submit or revert, go to client workspace"
1731             print "  " + self.clientPath
1732             print
1733             print "To submit, use \"p4 submit\" to write a new description,"
1734             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1735                   " \"git p4\"." % fileName
1736             print "You can delete the file \"%s\" when finished." % fileName
1737
1738             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1739                 print "To preserve change ownership by user %s, you must\n" \
1740                       "do \"p4 change -f <change>\" after submitting and\n" \
1741                       "edit the User field."
1742             if pureRenameCopy:
1743                 print "After submitting, renamed files must be re-synced."
1744                 print "Invoke \"p4 sync -f\" on each of these files:"
1745                 for f in pureRenameCopy:
1746                     print "  " + f
1747
1748             print
1749             print "To revert the changes, use \"p4 revert ...\", and delete"
1750             print "the submit template file \"%s\"" % fileName
1751             if filesToAdd:
1752                 print "Since the commit adds new files, they must be deleted:"
1753                 for f in filesToAdd:
1754                     print "  " + f
1755             print
1756             return True
1757
1758         #
1759         # Let the user edit the change description, then submit it.
1760         #
1761         submitted = False
1762
1763         try:
1764             if self.edit_template(fileName):
1765                 # read the edited message and submit
1766                 tmpFile = open(fileName, "rb")
1767                 message = tmpFile.read()
1768                 tmpFile.close()
1769                 if self.isWindows:
1770                     message = message.replace("\r\n", "\n")
1771                 submitTemplate = message[:message.index(separatorLine)]
1772                 p4_write_pipe(['submit', '-i'], submitTemplate)
1773
1774                 if self.preserveUser:
1775                     if p4User:
1776                         # Get last changelist number. Cannot easily get it from
1777                         # the submit command output as the output is
1778                         # unmarshalled.
1779                         changelist = self.lastP4Changelist()
1780                         self.modifyChangelistUser(changelist, p4User)
1781
1782                 # The rename/copy happened by applying a patch that created a
1783                 # new file.  This leaves it writable, which confuses p4.
1784                 for f in pureRenameCopy:
1785                     p4_sync(f, "-f")
1786                 submitted = True
1787
1788         finally:
1789             # skip this patch
1790             if not submitted:
1791                 print "Submission cancelled, undoing p4 changes."
1792                 for f in editedFiles:
1793                     p4_revert(f)
1794                 for f in filesToAdd:
1795                     p4_revert(f)
1796                     os.remove(f)
1797                 for f in filesToDelete:
1798                     p4_revert(f)
1799
1800         os.remove(fileName)
1801         return submitted
1802
1803     # Export git tags as p4 labels. Create a p4 label and then tag
1804     # with that.
1805     def exportGitTags(self, gitTags):
1806         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1807         if len(validLabelRegexp) == 0:
1808             validLabelRegexp = defaultLabelRegexp
1809         m = re.compile(validLabelRegexp)
1810
1811         for name in gitTags:
1812
1813             if not m.match(name):
1814                 if verbose:
1815                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1816                 continue
1817
1818             # Get the p4 commit this corresponds to
1819             logMessage = extractLogMessageFromGitCommit(name)
1820             values = extractSettingsGitLog(logMessage)
1821
1822             if not values.has_key('change'):
1823                 # a tag pointing to something not sent to p4; ignore
1824                 if verbose:
1825                     print "git tag %s does not give a p4 commit" % name
1826                 continue
1827             else:
1828                 changelist = values['change']
1829
1830             # Get the tag details.
1831             inHeader = True
1832             isAnnotated = False
1833             body = []
1834             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1835                 l = l.strip()
1836                 if inHeader:
1837                     if re.match(r'tag\s+', l):
1838                         isAnnotated = True
1839                     elif re.match(r'\s*$', l):
1840                         inHeader = False
1841                         continue
1842                 else:
1843                     body.append(l)
1844
1845             if not isAnnotated:
1846                 body = ["lightweight tag imported by git p4\n"]
1847
1848             # Create the label - use the same view as the client spec we are using
1849             clientSpec = getClientSpec()
1850
1851             labelTemplate  = "Label: %s\n" % name
1852             labelTemplate += "Description:\n"
1853             for b in body:
1854                 labelTemplate += "\t" + b + "\n"
1855             labelTemplate += "View:\n"
1856             for depot_side in clientSpec.mappings:
1857                 labelTemplate += "\t%s\n" % depot_side
1858
1859             if self.dry_run:
1860                 print "Would create p4 label %s for tag" % name
1861             elif self.prepare_p4_only:
1862                 print "Not creating p4 label %s for tag due to option" \
1863                       " --prepare-p4-only" % name
1864             else:
1865                 p4_write_pipe(["label", "-i"], labelTemplate)
1866
1867                 # Use the label
1868                 p4_system(["tag", "-l", name] +
1869                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1870
1871                 if verbose:
1872                     print "created p4 label for tag %s" % name
1873
1874     def run(self, args):
1875         if len(args) == 0:
1876             self.master = currentGitBranch()
1877         elif len(args) == 1:
1878             self.master = args[0]
1879             if not branchExists(self.master):
1880                 die("Branch %s does not exist" % self.master)
1881         else:
1882             return False
1883
1884         if self.master:
1885             allowSubmit = gitConfig("git-p4.allowSubmit")
1886             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1887                 die("%s is not in git-p4.allowSubmit" % self.master)
1888
1889         [upstream, settings] = findUpstreamBranchPoint()
1890         self.depotPath = settings['depot-paths'][0]
1891         if len(self.origin) == 0:
1892             self.origin = upstream
1893
1894         if self.preserveUser:
1895             if not self.canChangeChangelists():
1896                 die("Cannot preserve user names without p4 super-user or admin permissions")
1897
1898         # if not set from the command line, try the config file
1899         if self.conflict_behavior is None:
1900             val = gitConfig("git-p4.conflict")
1901             if val:
1902                 if val not in self.conflict_behavior_choices:
1903                     die("Invalid value '%s' for config git-p4.conflict" % val)
1904             else:
1905                 val = "ask"
1906             self.conflict_behavior = val
1907
1908         if self.verbose:
1909             print "Origin branch is " + self.origin
1910
1911         if len(self.depotPath) == 0:
1912             print "Internal error: cannot locate perforce depot path from existing branches"
1913             sys.exit(128)
1914
1915         self.useClientSpec = False
1916         if gitConfigBool("git-p4.useclientspec"):
1917             self.useClientSpec = True
1918         if self.useClientSpec:
1919             self.clientSpecDirs = getClientSpec()
1920
1921         # Check for the existance of P4 branches
1922         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1923
1924         if self.useClientSpec and not branchesDetected:
1925             # all files are relative to the client spec
1926             self.clientPath = getClientRoot()
1927         else:
1928             self.clientPath = p4Where(self.depotPath)
1929
1930         if self.clientPath == "":
1931             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1932
1933         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1934         self.oldWorkingDirectory = os.getcwd()
1935
1936         # ensure the clientPath exists
1937         new_client_dir = False
1938         if not os.path.exists(self.clientPath):
1939             new_client_dir = True
1940             os.makedirs(self.clientPath)
1941
1942         chdir(self.clientPath, is_client_path=True)
1943         if self.dry_run:
1944             print "Would synchronize p4 checkout in %s" % self.clientPath
1945         else:
1946             print "Synchronizing p4 checkout..."
1947             if new_client_dir:
1948                 # old one was destroyed, and maybe nobody told p4
1949                 p4_sync("...", "-f")
1950             else:
1951                 p4_sync("...")
1952         self.check()
1953
1954         commits = []
1955         if self.master:
1956             commitish = self.master
1957         else:
1958             commitish = 'HEAD'
1959
1960         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1961             commits.append(line.strip())
1962         commits.reverse()
1963
1964         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1965             self.checkAuthorship = False
1966         else:
1967             self.checkAuthorship = True
1968
1969         if self.preserveUser:
1970             self.checkValidP4Users(commits)
1971
1972         #
1973         # Build up a set of options to be passed to diff when
1974         # submitting each commit to p4.
1975         #
1976         if self.detectRenames:
1977             # command-line -M arg
1978             self.diffOpts = "-M"
1979         else:
1980             # If not explicitly set check the config variable
1981             detectRenames = gitConfig("git-p4.detectRenames")
1982
1983             if detectRenames.lower() == "false" or detectRenames == "":
1984                 self.diffOpts = ""
1985             elif detectRenames.lower() == "true":
1986                 self.diffOpts = "-M"
1987             else:
1988                 self.diffOpts = "-M%s" % detectRenames
1989
1990         # no command-line arg for -C or --find-copies-harder, just
1991         # config variables
1992         detectCopies = gitConfig("git-p4.detectCopies")
1993         if detectCopies.lower() == "false" or detectCopies == "":
1994             pass
1995         elif detectCopies.lower() == "true":
1996             self.diffOpts += " -C"
1997         else:
1998             self.diffOpts += " -C%s" % detectCopies
1999
2000         if gitConfigBool("git-p4.detectCopiesHarder"):
2001             self.diffOpts += " --find-copies-harder"
2002
2003         #
2004         # Apply the commits, one at a time.  On failure, ask if should
2005         # continue to try the rest of the patches, or quit.
2006         #
2007         if self.dry_run:
2008             print "Would apply"
2009         applied = []
2010         last = len(commits) - 1
2011         for i, commit in enumerate(commits):
2012             if self.dry_run:
2013                 print " ", read_pipe(["git", "show", "-s",
2014                                       "--format=format:%h %s", commit])
2015                 ok = True
2016             else:
2017                 ok = self.applyCommit(commit)
2018             if ok:
2019                 applied.append(commit)
2020             else:
2021                 if self.prepare_p4_only and i < last:
2022                     print "Processing only the first commit due to option" \
2023                           " --prepare-p4-only"
2024                     break
2025                 if i < last:
2026                     quit = False
2027                     while True:
2028                         # prompt for what to do, or use the option/variable
2029                         if self.conflict_behavior == "ask":
2030                             print "What do you want to do?"
2031                             response = raw_input("[s]kip this commit but apply"
2032                                                  " the rest, or [q]uit? ")
2033                             if not response:
2034                                 continue
2035                         elif self.conflict_behavior == "skip":
2036                             response = "s"
2037                         elif self.conflict_behavior == "quit":
2038                             response = "q"
2039                         else:
2040                             die("Unknown conflict_behavior '%s'" %
2041                                 self.conflict_behavior)
2042
2043                         if response[0] == "s":
2044                             print "Skipping this commit, but applying the rest"
2045                             break
2046                         if response[0] == "q":
2047                             print "Quitting"
2048                             quit = True
2049                             break
2050                     if quit:
2051                         break
2052
2053         chdir(self.oldWorkingDirectory)
2054
2055         if self.dry_run:
2056             pass
2057         elif self.prepare_p4_only:
2058             pass
2059         elif len(commits) == len(applied):
2060             print "All commits applied!"
2061
2062             sync = P4Sync()
2063             if self.branch:
2064                 sync.branch = self.branch
2065             sync.run([])
2066
2067             rebase = P4Rebase()
2068             rebase.rebase()
2069
2070         else:
2071             if len(applied) == 0:
2072                 print "No commits applied."
2073             else:
2074                 print "Applied only the commits marked with '*':"
2075                 for c in commits:
2076                     if c in applied:
2077                         star = "*"
2078                     else:
2079                         star = " "
2080                     print star, read_pipe(["git", "show", "-s",
2081                                            "--format=format:%h %s",  c])
2082                 print "You will have to do 'git p4 sync' and rebase."
2083
2084         if gitConfigBool("git-p4.exportLabels"):
2085             self.exportLabels = True
2086
2087         if self.exportLabels:
2088             p4Labels = getP4Labels(self.depotPath)
2089             gitTags = getGitTags()
2090
2091             missingGitTags = gitTags - p4Labels
2092             self.exportGitTags(missingGitTags)
2093
2094         # exit with error unless everything applied perfectly
2095         if len(commits) != len(applied):
2096                 sys.exit(1)
2097
2098         return True
2099
2100 class View(object):
2101     """Represent a p4 view ("p4 help views"), and map files in a
2102        repo according to the view."""
2103
2104     def __init__(self, client_name):
2105         self.mappings = []
2106         self.client_prefix = "//%s/" % client_name
2107         # cache results of "p4 where" to lookup client file locations
2108         self.client_spec_path_cache = {}
2109
2110     def append(self, view_line):
2111         """Parse a view line, splitting it into depot and client
2112            sides.  Append to self.mappings, preserving order.  This
2113            is only needed for tag creation."""
2114
2115         # Split the view line into exactly two words.  P4 enforces
2116         # structure on these lines that simplifies this quite a bit.
2117         #
2118         # Either or both words may be double-quoted.
2119         # Single quotes do not matter.
2120         # Double-quote marks cannot occur inside the words.
2121         # A + or - prefix is also inside the quotes.
2122         # There are no quotes unless they contain a space.
2123         # The line is already white-space stripped.
2124         # The two words are separated by a single space.
2125         #
2126         if view_line[0] == '"':
2127             # First word is double quoted.  Find its end.
2128             close_quote_index = view_line.find('"', 1)
2129             if close_quote_index <= 0:
2130                 die("No first-word closing quote found: %s" % view_line)
2131             depot_side = view_line[1:close_quote_index]
2132             # skip closing quote and space
2133             rhs_index = close_quote_index + 1 + 1
2134         else:
2135             space_index = view_line.find(" ")
2136             if space_index <= 0:
2137                 die("No word-splitting space found: %s" % view_line)
2138             depot_side = view_line[0:space_index]
2139             rhs_index = space_index + 1
2140
2141         # prefix + means overlay on previous mapping
2142         if depot_side.startswith("+"):
2143             depot_side = depot_side[1:]
2144
2145         # prefix - means exclude this path, leave out of mappings
2146         exclude = False
2147         if depot_side.startswith("-"):
2148             exclude = True
2149             depot_side = depot_side[1:]
2150
2151         if not exclude:
2152             self.mappings.append(depot_side)
2153
2154     def convert_client_path(self, clientFile):
2155         # chop off //client/ part to make it relative
2156         if not clientFile.startswith(self.client_prefix):
2157             die("No prefix '%s' on clientFile '%s'" %
2158                 (self.client_prefix, clientFile))
2159         return clientFile[len(self.client_prefix):]
2160
2161     def update_client_spec_path_cache(self, files):
2162         """ Caching file paths by "p4 where" batch query """
2163
2164         # List depot file paths exclude that already cached
2165         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2166
2167         if len(fileArgs) == 0:
2168             return  # All files in cache
2169
2170         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2171         for res in where_result:
2172             if "code" in res and res["code"] == "error":
2173                 # assume error is "... file(s) not in client view"
2174                 continue
2175             if "clientFile" not in res:
2176                 die("No clientFile in 'p4 where' output")
2177             if "unmap" in res:
2178                 # it will list all of them, but only one not unmap-ped
2179                 continue
2180             if gitConfigBool("core.ignorecase"):
2181                 res['depotFile'] = res['depotFile'].lower()
2182             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2183
2184         # not found files or unmap files set to ""
2185         for depotFile in fileArgs:
2186             if gitConfigBool("core.ignorecase"):
2187                 depotFile = depotFile.lower()
2188             if depotFile not in self.client_spec_path_cache:
2189                 self.client_spec_path_cache[depotFile] = ""
2190
2191     def map_in_client(self, depot_path):
2192         """Return the relative location in the client where this
2193            depot file should live.  Returns "" if the file should
2194            not be mapped in the client."""
2195
2196         if gitConfigBool("core.ignorecase"):
2197             depot_path = depot_path.lower()
2198
2199         if depot_path in self.client_spec_path_cache:
2200             return self.client_spec_path_cache[depot_path]
2201
2202         die( "Error: %s is not found in client spec path" % depot_path )
2203         return ""
2204
2205 class P4Sync(Command, P4UserMap):
2206     delete_actions = ( "delete", "move/delete", "purge" )
2207
2208     def __init__(self):
2209         Command.__init__(self)
2210         P4UserMap.__init__(self)
2211         self.options = [
2212                 optparse.make_option("--branch", dest="branch"),
2213                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2214                 optparse.make_option("--changesfile", dest="changesFile"),
2215                 optparse.make_option("--silent", dest="silent", action="store_true"),
2216                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2217                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2218                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2219                                      help="Import into refs/heads/ , not refs/remotes"),
2220                 optparse.make_option("--max-changes", dest="maxChanges",
2221                                      help="Maximum number of changes to import"),
2222                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2223                                      help="Internal block size to use when iteratively calling p4 changes"),
2224                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2225                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2226                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2227                                      help="Only sync files that are included in the Perforce Client Spec"),
2228                 optparse.make_option("-/", dest="cloneExclude",
2229                                      action="append", type="string",
2230                                      help="exclude depot path"),
2231         ]
2232         self.description = """Imports from Perforce into a git repository.\n
2233     example:
2234     //depot/my/project/ -- to import the current head
2235     //depot/my/project/@all -- to import everything
2236     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2237
2238     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2239
2240         self.usage += " //depot/path[@revRange]"
2241         self.silent = False
2242         self.createdBranches = set()
2243         self.committedChanges = set()
2244         self.branch = ""
2245         self.detectBranches = False
2246         self.detectLabels = False
2247         self.importLabels = False
2248         self.changesFile = ""
2249         self.syncWithOrigin = True
2250         self.importIntoRemotes = True
2251         self.maxChanges = ""
2252         self.changes_block_size = None
2253         self.keepRepoPath = False
2254         self.depotPaths = None
2255         self.p4BranchesInGit = []
2256         self.cloneExclude = []
2257         self.useClientSpec = False
2258         self.useClientSpec_from_options = False
2259         self.clientSpecDirs = None
2260         self.tempBranches = []
2261         self.tempBranchLocation = "git-p4-tmp"
2262         self.largeFileSystem = None
2263
2264         if gitConfig('git-p4.largeFileSystem'):
2265             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2266             self.largeFileSystem = largeFileSystemConstructor(
2267                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2268             )
2269
2270         if gitConfig("git-p4.syncFromOrigin") == "false":
2271             self.syncWithOrigin = False
2272
2273     # This is required for the "append" cloneExclude action
2274     def ensure_value(self, attr, value):
2275         if not hasattr(self, attr) or getattr(self, attr) is None:
2276             setattr(self, attr, value)
2277         return getattr(self, attr)
2278
2279     # Force a checkpoint in fast-import and wait for it to finish
2280     def checkpoint(self):
2281         self.gitStream.write("checkpoint\n\n")
2282         self.gitStream.write("progress checkpoint\n\n")
2283         out = self.gitOutput.readline()
2284         if self.verbose:
2285             print "checkpoint finished: " + out
2286
2287     def extractFilesFromCommit(self, commit):
2288         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2289                              for path in self.cloneExclude]
2290         files = []
2291         fnum = 0
2292         while commit.has_key("depotFile%s" % fnum):
2293             path =  commit["depotFile%s" % fnum]
2294
2295             if [p for p in self.cloneExclude
2296                 if p4PathStartsWith(path, p)]:
2297                 found = False
2298             else:
2299                 found = [p for p in self.depotPaths
2300                          if p4PathStartsWith(path, p)]
2301             if not found:
2302                 fnum = fnum + 1
2303                 continue
2304
2305             file = {}
2306             file["path"] = path
2307             file["rev"] = commit["rev%s" % fnum]
2308             file["action"] = commit["action%s" % fnum]
2309             file["type"] = commit["type%s" % fnum]
2310             files.append(file)
2311             fnum = fnum + 1
2312         return files
2313
2314     def stripRepoPath(self, path, prefixes):
2315         """When streaming files, this is called to map a p4 depot path
2316            to where it should go in git.  The prefixes are either
2317            self.depotPaths, or self.branchPrefixes in the case of
2318            branch detection."""
2319
2320         if self.useClientSpec:
2321             # branch detection moves files up a level (the branch name)
2322             # from what client spec interpretation gives
2323             path = self.clientSpecDirs.map_in_client(path)
2324             if self.detectBranches:
2325                 for b in self.knownBranches:
2326                     if path.startswith(b + "/"):
2327                         path = path[len(b)+1:]
2328
2329         elif self.keepRepoPath:
2330             # Preserve everything in relative path name except leading
2331             # //depot/; just look at first prefix as they all should
2332             # be in the same depot.
2333             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2334             if p4PathStartsWith(path, depot):
2335                 path = path[len(depot):]
2336
2337         else:
2338             for p in prefixes:
2339                 if p4PathStartsWith(path, p):
2340                     path = path[len(p):]
2341                     break
2342
2343         path = wildcard_decode(path)
2344         return path
2345
2346     def splitFilesIntoBranches(self, commit):
2347         """Look at each depotFile in the commit to figure out to what
2348            branch it belongs."""
2349
2350         if self.clientSpecDirs:
2351             files = self.extractFilesFromCommit(commit)
2352             self.clientSpecDirs.update_client_spec_path_cache(files)
2353
2354         branches = {}
2355         fnum = 0
2356         while commit.has_key("depotFile%s" % fnum):
2357             path =  commit["depotFile%s" % fnum]
2358             found = [p for p in self.depotPaths
2359                      if p4PathStartsWith(path, p)]
2360             if not found:
2361                 fnum = fnum + 1
2362                 continue
2363
2364             file = {}
2365             file["path"] = path
2366             file["rev"] = commit["rev%s" % fnum]
2367             file["action"] = commit["action%s" % fnum]
2368             file["type"] = commit["type%s" % fnum]
2369             fnum = fnum + 1
2370
2371             # start with the full relative path where this file would
2372             # go in a p4 client
2373             if self.useClientSpec:
2374                 relPath = self.clientSpecDirs.map_in_client(path)
2375             else:
2376                 relPath = self.stripRepoPath(path, self.depotPaths)
2377
2378             for branch in self.knownBranches.keys():
2379                 # add a trailing slash so that a commit into qt/4.2foo
2380                 # doesn't end up in qt/4.2, e.g.
2381                 if relPath.startswith(branch + "/"):
2382                     if branch not in branches:
2383                         branches[branch] = []
2384                     branches[branch].append(file)
2385                     break
2386
2387         return branches
2388
2389     def writeToGitStream(self, gitMode, relPath, contents):
2390         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2391         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2392         for d in contents:
2393             self.gitStream.write(d)
2394         self.gitStream.write('\n')
2395
2396     # output one file from the P4 stream
2397     # - helper for streamP4Files
2398
2399     def streamOneP4File(self, file, contents):
2400         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2401         if verbose:
2402             size = int(self.stream_file['fileSize'])
2403             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2404             sys.stdout.flush()
2405
2406         (type_base, type_mods) = split_p4_type(file["type"])
2407
2408         git_mode = "100644"
2409         if "x" in type_mods:
2410             git_mode = "100755"
2411         if type_base == "symlink":
2412             git_mode = "120000"
2413             # p4 print on a symlink sometimes contains "target\n";
2414             # if it does, remove the newline
2415             data = ''.join(contents)
2416             if not data:
2417                 # Some version of p4 allowed creating a symlink that pointed
2418                 # to nothing.  This causes p4 errors when checking out such
2419                 # a change, and errors here too.  Work around it by ignoring
2420                 # the bad symlink; hopefully a future change fixes it.
2421                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2422                 return
2423             elif data[-1] == '\n':
2424                 contents = [data[:-1]]
2425             else:
2426                 contents = [data]
2427
2428         if type_base == "utf16":
2429             # p4 delivers different text in the python output to -G
2430             # than it does when using "print -o", or normal p4 client
2431             # operations.  utf16 is converted to ascii or utf8, perhaps.
2432             # But ascii text saved as -t utf16 is completely mangled.
2433             # Invoke print -o to get the real contents.
2434             #
2435             # On windows, the newlines will always be mangled by print, so put
2436             # them back too.  This is not needed to the cygwin windows version,
2437             # just the native "NT" type.
2438             #
2439             try:
2440                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2441             except Exception as e:
2442                 if 'Translation of file content failed' in str(e):
2443                     type_base = 'binary'
2444                 else:
2445                     raise e
2446             else:
2447                 if p4_version_string().find('/NT') >= 0:
2448                     text = text.replace('\r\n', '\n')
2449                 contents = [ text ]
2450
2451         if type_base == "apple":
2452             # Apple filetype files will be streamed as a concatenation of
2453             # its appledouble header and the contents.  This is useless
2454             # on both macs and non-macs.  If using "print -q -o xx", it
2455             # will create "xx" with the data, and "%xx" with the header.
2456             # This is also not very useful.
2457             #
2458             # Ideally, someday, this script can learn how to generate
2459             # appledouble files directly and import those to git, but
2460             # non-mac machines can never find a use for apple filetype.
2461             print "\nIgnoring apple filetype file %s" % file['depotFile']
2462             return
2463
2464         # Note that we do not try to de-mangle keywords on utf16 files,
2465         # even though in theory somebody may want that.
2466         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2467         if pattern:
2468             regexp = re.compile(pattern, re.VERBOSE)
2469             text = ''.join(contents)
2470             text = regexp.sub(r'$\1$', text)
2471             contents = [ text ]
2472
2473         try:
2474             relPath.decode('ascii')
2475         except:
2476             encoding = 'utf8'
2477             if gitConfig('git-p4.pathEncoding'):
2478                 encoding = gitConfig('git-p4.pathEncoding')
2479             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2480             if self.verbose:
2481                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2482
2483         if self.largeFileSystem:
2484             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2485
2486         self.writeToGitStream(git_mode, relPath, contents)
2487
2488     def streamOneP4Deletion(self, file):
2489         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2490         if verbose:
2491             sys.stdout.write("delete %s\n" % relPath)
2492             sys.stdout.flush()
2493         self.gitStream.write("D %s\n" % relPath)
2494
2495         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2496             self.largeFileSystem.removeLargeFile(relPath)
2497
2498     # handle another chunk of streaming data
2499     def streamP4FilesCb(self, marshalled):
2500
2501         # catch p4 errors and complain
2502         err = None
2503         if "code" in marshalled:
2504             if marshalled["code"] == "error":
2505                 if "data" in marshalled:
2506                     err = marshalled["data"].rstrip()
2507
2508         if not err and 'fileSize' in self.stream_file:
2509             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2510             if required_bytes > 0:
2511                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2512                     os.getcwd(), required_bytes/1024/1024
2513                 )
2514
2515         if err:
2516             f = None
2517             if self.stream_have_file_info:
2518                 if "depotFile" in self.stream_file:
2519                     f = self.stream_file["depotFile"]
2520             # force a failure in fast-import, else an empty
2521             # commit will be made
2522             self.gitStream.write("\n")
2523             self.gitStream.write("die-now\n")
2524             self.gitStream.close()
2525             # ignore errors, but make sure it exits first
2526             self.importProcess.wait()
2527             if f:
2528                 die("Error from p4 print for %s: %s" % (f, err))
2529             else:
2530                 die("Error from p4 print: %s" % err)
2531
2532         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2533             # start of a new file - output the old one first
2534             self.streamOneP4File(self.stream_file, self.stream_contents)
2535             self.stream_file = {}
2536             self.stream_contents = []
2537             self.stream_have_file_info = False
2538
2539         # pick up the new file information... for the
2540         # 'data' field we need to append to our array
2541         for k in marshalled.keys():
2542             if k == 'data':
2543                 if 'streamContentSize' not in self.stream_file:
2544                     self.stream_file['streamContentSize'] = 0
2545                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2546                 self.stream_contents.append(marshalled['data'])
2547             else:
2548                 self.stream_file[k] = marshalled[k]
2549
2550         if (verbose and
2551             'streamContentSize' in self.stream_file and
2552             'fileSize' in self.stream_file and
2553             'depotFile' in self.stream_file):
2554             size = int(self.stream_file["fileSize"])
2555             if size > 0:
2556                 progress = 100*self.stream_file['streamContentSize']/size
2557                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2558                 sys.stdout.flush()
2559
2560         self.stream_have_file_info = True
2561
2562     # Stream directly from "p4 files" into "git fast-import"
2563     def streamP4Files(self, files):
2564         filesForCommit = []
2565         filesToRead = []
2566         filesToDelete = []
2567
2568         for f in files:
2569             filesForCommit.append(f)
2570             if f['action'] in self.delete_actions:
2571                 filesToDelete.append(f)
2572             else:
2573                 filesToRead.append(f)
2574
2575         # deleted files...
2576         for f in filesToDelete:
2577             self.streamOneP4Deletion(f)
2578
2579         if len(filesToRead) > 0:
2580             self.stream_file = {}
2581             self.stream_contents = []
2582             self.stream_have_file_info = False
2583
2584             # curry self argument
2585             def streamP4FilesCbSelf(entry):
2586                 self.streamP4FilesCb(entry)
2587
2588             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2589
2590             p4CmdList(["-x", "-", "print"],
2591                       stdin=fileArgs,
2592                       cb=streamP4FilesCbSelf)
2593
2594             # do the last chunk
2595             if self.stream_file.has_key('depotFile'):
2596                 self.streamOneP4File(self.stream_file, self.stream_contents)
2597
2598     def make_email(self, userid):
2599         if userid in self.users:
2600             return self.users[userid]
2601         else:
2602             return "%s <a@b>" % userid
2603
2604     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2605         """ Stream a p4 tag.
2606         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2607         """
2608
2609         if verbose:
2610             print "writing tag %s for commit %s" % (labelName, commit)
2611         gitStream.write("tag %s\n" % labelName)
2612         gitStream.write("from %s\n" % commit)
2613
2614         if labelDetails.has_key('Owner'):
2615             owner = labelDetails["Owner"]
2616         else:
2617             owner = None
2618
2619         # Try to use the owner of the p4 label, or failing that,
2620         # the current p4 user id.
2621         if owner:
2622             email = self.make_email(owner)
2623         else:
2624             email = self.make_email(self.p4UserId())
2625         tagger = "%s %s %s" % (email, epoch, self.tz)
2626
2627         gitStream.write("tagger %s\n" % tagger)
2628
2629         print "labelDetails=",labelDetails
2630         if labelDetails.has_key('Description'):
2631             description = labelDetails['Description']
2632         else:
2633             description = 'Label from git p4'
2634
2635         gitStream.write("data %d\n" % len(description))
2636         gitStream.write(description)
2637         gitStream.write("\n")
2638
2639     def inClientSpec(self, path):
2640         if not self.clientSpecDirs:
2641             return True
2642         inClientSpec = self.clientSpecDirs.map_in_client(path)
2643         if not inClientSpec and self.verbose:
2644             print('Ignoring file outside of client spec: {0}'.format(path))
2645         return inClientSpec
2646
2647     def hasBranchPrefix(self, path):
2648         if not self.branchPrefixes:
2649             return True
2650         hasPrefix = [p for p in self.branchPrefixes
2651                         if p4PathStartsWith(path, p)]
2652         if hasPrefix and self.verbose:
2653             print('Ignoring file outside of prefix: {0}'.format(path))
2654         return hasPrefix
2655
2656     def commit(self, details, files, branch, parent = ""):
2657         epoch = details["time"]
2658         author = details["user"]
2659
2660         if self.verbose:
2661             print('commit into {0}'.format(branch))
2662
2663         if self.clientSpecDirs:
2664             self.clientSpecDirs.update_client_spec_path_cache(files)
2665
2666         files = [f for f in files
2667             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2668
2669         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2670             print('Ignoring revision {0} as it would produce an empty commit.'
2671                 .format(details['change']))
2672             return
2673
2674         self.gitStream.write("commit %s\n" % branch)
2675         self.gitStream.write("mark :%s\n" % details["change"])
2676         self.committedChanges.add(int(details["change"]))
2677         committer = ""
2678         if author not in self.users:
2679             self.getUserMapFromPerforceServer()
2680         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2681
2682         self.gitStream.write("committer %s\n" % committer)
2683
2684         self.gitStream.write("data <<EOT\n")
2685         self.gitStream.write(details["desc"])
2686         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2687                              (','.join(self.branchPrefixes), details["change"]))
2688         if len(details['options']) > 0:
2689             self.gitStream.write(": options = %s" % details['options'])
2690         self.gitStream.write("]\nEOT\n\n")
2691
2692         if len(parent) > 0:
2693             if self.verbose:
2694                 print "parent %s" % parent
2695             self.gitStream.write("from %s\n" % parent)
2696
2697         self.streamP4Files(files)
2698         self.gitStream.write("\n")
2699
2700         change = int(details["change"])
2701
2702         if self.labels.has_key(change):
2703             label = self.labels[change]
2704             labelDetails = label[0]
2705             labelRevisions = label[1]
2706             if self.verbose:
2707                 print "Change %s is labelled %s" % (change, labelDetails)
2708
2709             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2710                                                 for p in self.branchPrefixes])
2711
2712             if len(files) == len(labelRevisions):
2713
2714                 cleanedFiles = {}
2715                 for info in files:
2716                     if info["action"] in self.delete_actions:
2717                         continue
2718                     cleanedFiles[info["depotFile"]] = info["rev"]
2719
2720                 if cleanedFiles == labelRevisions:
2721                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2722
2723                 else:
2724                     if not self.silent:
2725                         print ("Tag %s does not match with change %s: files do not match."
2726                                % (labelDetails["label"], change))
2727
2728             else:
2729                 if not self.silent:
2730                     print ("Tag %s does not match with change %s: file count is different."
2731                            % (labelDetails["label"], change))
2732
2733     # Build a dictionary of changelists and labels, for "detect-labels" option.
2734     def getLabels(self):
2735         self.labels = {}
2736
2737         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2738         if len(l) > 0 and not self.silent:
2739             print "Finding files belonging to labels in %s" % `self.depotPaths`
2740
2741         for output in l:
2742             label = output["label"]
2743             revisions = {}
2744             newestChange = 0
2745             if self.verbose:
2746                 print "Querying files for label %s" % label
2747             for file in p4CmdList(["files"] +
2748                                       ["%s...@%s" % (p, label)
2749                                           for p in self.depotPaths]):
2750                 revisions[file["depotFile"]] = file["rev"]
2751                 change = int(file["change"])
2752                 if change > newestChange:
2753                     newestChange = change
2754
2755             self.labels[newestChange] = [output, revisions]
2756
2757         if self.verbose:
2758             print "Label changes: %s" % self.labels.keys()
2759
2760     # Import p4 labels as git tags. A direct mapping does not
2761     # exist, so assume that if all the files are at the same revision
2762     # then we can use that, or it's something more complicated we should
2763     # just ignore.
2764     def importP4Labels(self, stream, p4Labels):
2765         if verbose:
2766             print "import p4 labels: " + ' '.join(p4Labels)
2767
2768         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2769         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2770         if len(validLabelRegexp) == 0:
2771             validLabelRegexp = defaultLabelRegexp
2772         m = re.compile(validLabelRegexp)
2773
2774         for name in p4Labels:
2775             commitFound = False
2776
2777             if not m.match(name):
2778                 if verbose:
2779                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2780                 continue
2781
2782             if name in ignoredP4Labels:
2783                 continue
2784
2785             labelDetails = p4CmdList(['label', "-o", name])[0]
2786
2787             # get the most recent changelist for each file in this label
2788             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2789                                 for p in self.depotPaths])
2790
2791             if change.has_key('change'):
2792                 # find the corresponding git commit; take the oldest commit
2793                 changelist = int(change['change'])
2794                 if changelist in self.committedChanges:
2795                     gitCommit = ":%d" % changelist       # use a fast-import mark
2796                     commitFound = True
2797                 else:
2798                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2799                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2800                     if len(gitCommit) == 0:
2801                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2802                     else:
2803                         commitFound = True
2804                         gitCommit = gitCommit.strip()
2805
2806                 if commitFound:
2807                     # Convert from p4 time format
2808                     try:
2809                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2810                     except ValueError:
2811                         print "Could not convert label time %s" % labelDetails['Update']
2812                         tmwhen = 1
2813
2814                     when = int(time.mktime(tmwhen))
2815                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2816                     if verbose:
2817                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2818             else:
2819                 if verbose:
2820                     print "Label %s has no changelists - possibly deleted?" % name
2821
2822             if not commitFound:
2823                 # We can't import this label; don't try again as it will get very
2824                 # expensive repeatedly fetching all the files for labels that will
2825                 # never be imported. If the label is moved in the future, the
2826                 # ignore will need to be removed manually.
2827                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2828
2829     def guessProjectName(self):
2830         for p in self.depotPaths:
2831             if p.endswith("/"):
2832                 p = p[:-1]
2833             p = p[p.strip().rfind("/") + 1:]
2834             if not p.endswith("/"):
2835                p += "/"
2836             return p
2837
2838     def getBranchMapping(self):
2839         lostAndFoundBranches = set()
2840
2841         user = gitConfig("git-p4.branchUser")
2842         if len(user) > 0:
2843             command = "branches -u %s" % user
2844         else:
2845             command = "branches"
2846
2847         for info in p4CmdList(command):
2848             details = p4Cmd(["branch", "-o", info["branch"]])
2849             viewIdx = 0
2850             while details.has_key("View%s" % viewIdx):
2851                 paths = details["View%s" % viewIdx].split(" ")
2852                 viewIdx = viewIdx + 1
2853                 # require standard //depot/foo/... //depot/bar/... mapping
2854                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2855                     continue
2856                 source = paths[0]
2857                 destination = paths[1]
2858                 ## HACK
2859                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2860                     source = source[len(self.depotPaths[0]):-4]
2861                     destination = destination[len(self.depotPaths[0]):-4]
2862
2863                     if destination in self.knownBranches:
2864                         if not self.silent:
2865                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2866                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2867                         continue
2868
2869                     self.knownBranches[destination] = source
2870
2871                     lostAndFoundBranches.discard(destination)
2872
2873                     if source not in self.knownBranches:
2874                         lostAndFoundBranches.add(source)
2875
2876         # Perforce does not strictly require branches to be defined, so we also
2877         # check git config for a branch list.
2878         #
2879         # Example of branch definition in git config file:
2880         # [git-p4]
2881         #   branchList=main:branchA
2882         #   branchList=main:branchB
2883         #   branchList=branchA:branchC
2884         configBranches = gitConfigList("git-p4.branchList")
2885         for branch in configBranches:
2886             if branch:
2887                 (source, destination) = branch.split(":")
2888                 self.knownBranches[destination] = source
2889
2890                 lostAndFoundBranches.discard(destination)
2891
2892                 if source not in self.knownBranches:
2893                     lostAndFoundBranches.add(source)
2894
2895
2896         for branch in lostAndFoundBranches:
2897             self.knownBranches[branch] = branch
2898
2899     def getBranchMappingFromGitBranches(self):
2900         branches = p4BranchesInGit(self.importIntoRemotes)
2901         for branch in branches.keys():
2902             if branch == "master":
2903                 branch = "main"
2904             else:
2905                 branch = branch[len(self.projectName):]
2906             self.knownBranches[branch] = branch
2907
2908     def updateOptionDict(self, d):
2909         option_keys = {}
2910         if self.keepRepoPath:
2911             option_keys['keepRepoPath'] = 1
2912
2913         d["options"] = ' '.join(sorted(option_keys.keys()))
2914
2915     def readOptions(self, d):
2916         self.keepRepoPath = (d.has_key('options')
2917                              and ('keepRepoPath' in d['options']))
2918
2919     def gitRefForBranch(self, branch):
2920         if branch == "main":
2921             return self.refPrefix + "master"
2922
2923         if len(branch) <= 0:
2924             return branch
2925
2926         return self.refPrefix + self.projectName + branch
2927
2928     def gitCommitByP4Change(self, ref, change):
2929         if self.verbose:
2930             print "looking in ref " + ref + " for change %s using bisect..." % change
2931
2932         earliestCommit = ""
2933         latestCommit = parseRevision(ref)
2934
2935         while True:
2936             if self.verbose:
2937                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2938             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2939             if len(next) == 0:
2940                 if self.verbose:
2941                     print "argh"
2942                 return ""
2943             log = extractLogMessageFromGitCommit(next)
2944             settings = extractSettingsGitLog(log)
2945             currentChange = int(settings['change'])
2946             if self.verbose:
2947                 print "current change %s" % currentChange
2948
2949             if currentChange == change:
2950                 if self.verbose:
2951                     print "found %s" % next
2952                 return next
2953
2954             if currentChange < change:
2955                 earliestCommit = "^%s" % next
2956             else:
2957                 latestCommit = "%s" % next
2958
2959         return ""
2960
2961     def importNewBranch(self, branch, maxChange):
2962         # make fast-import flush all changes to disk and update the refs using the checkpoint
2963         # command so that we can try to find the branch parent in the git history
2964         self.gitStream.write("checkpoint\n\n");
2965         self.gitStream.flush();
2966         branchPrefix = self.depotPaths[0] + branch + "/"
2967         range = "@1,%s" % maxChange
2968         #print "prefix" + branchPrefix
2969         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2970         if len(changes) <= 0:
2971             return False
2972         firstChange = changes[0]
2973         #print "first change in branch: %s" % firstChange
2974         sourceBranch = self.knownBranches[branch]
2975         sourceDepotPath = self.depotPaths[0] + sourceBranch
2976         sourceRef = self.gitRefForBranch(sourceBranch)
2977         #print "source " + sourceBranch
2978
2979         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2980         #print "branch parent: %s" % branchParentChange
2981         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2982         if len(gitParent) > 0:
2983             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2984             #print "parent git commit: %s" % gitParent
2985
2986         self.importChanges(changes)
2987         return True
2988
2989     def searchParent(self, parent, branch, target):
2990         parentFound = False
2991         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2992                                      "--no-merges", parent]):
2993             blob = blob.strip()
2994             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2995                 parentFound = True
2996                 if self.verbose:
2997                     print "Found parent of %s in commit %s" % (branch, blob)
2998                 break
2999         if parentFound:
3000             return blob
3001         else:
3002             return None
3003
3004     def importChanges(self, changes):
3005         cnt = 1
3006         for change in changes:
3007             description = p4_describe(change)
3008             self.updateOptionDict(description)
3009
3010             if not self.silent:
3011                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3012                 sys.stdout.flush()
3013             cnt = cnt + 1
3014
3015             try:
3016                 if self.detectBranches:
3017                     branches = self.splitFilesIntoBranches(description)
3018                     for branch in branches.keys():
3019                         ## HACK  --hwn
3020                         branchPrefix = self.depotPaths[0] + branch + "/"
3021                         self.branchPrefixes = [ branchPrefix ]
3022
3023                         parent = ""
3024
3025                         filesForCommit = branches[branch]
3026
3027                         if self.verbose:
3028                             print "branch is %s" % branch
3029
3030                         self.updatedBranches.add(branch)
3031
3032                         if branch not in self.createdBranches:
3033                             self.createdBranches.add(branch)
3034                             parent = self.knownBranches[branch]
3035                             if parent == branch:
3036                                 parent = ""
3037                             else:
3038                                 fullBranch = self.projectName + branch
3039                                 if fullBranch not in self.p4BranchesInGit:
3040                                     if not self.silent:
3041                                         print("\n    Importing new branch %s" % fullBranch);
3042                                     if self.importNewBranch(branch, change - 1):
3043                                         parent = ""
3044                                         self.p4BranchesInGit.append(fullBranch)
3045                                     if not self.silent:
3046                                         print("\n    Resuming with change %s" % change);
3047
3048                                 if self.verbose:
3049                                     print "parent determined through known branches: %s" % parent
3050
3051                         branch = self.gitRefForBranch(branch)
3052                         parent = self.gitRefForBranch(parent)
3053
3054                         if self.verbose:
3055                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3056
3057                         if len(parent) == 0 and branch in self.initialParents:
3058                             parent = self.initialParents[branch]
3059                             del self.initialParents[branch]
3060
3061                         blob = None
3062                         if len(parent) > 0:
3063                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3064                             if self.verbose:
3065                                 print "Creating temporary branch: " + tempBranch
3066                             self.commit(description, filesForCommit, tempBranch)
3067                             self.tempBranches.append(tempBranch)
3068                             self.checkpoint()
3069                             blob = self.searchParent(parent, branch, tempBranch)
3070                         if blob:
3071                             self.commit(description, filesForCommit, branch, blob)
3072                         else:
3073                             if self.verbose:
3074                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3075                             self.commit(description, filesForCommit, branch, parent)
3076                 else:
3077                     files = self.extractFilesFromCommit(description)
3078                     self.commit(description, files, self.branch,
3079                                 self.initialParent)
3080                     # only needed once, to connect to the previous commit
3081                     self.initialParent = ""
3082             except IOError:
3083                 print self.gitError.read()
3084                 sys.exit(1)
3085
3086     def importHeadRevision(self, revision):
3087         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3088
3089         details = {}
3090         details["user"] = "git perforce import user"
3091         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3092                            % (' '.join(self.depotPaths), revision))
3093         details["change"] = revision
3094         newestRevision = 0
3095
3096         fileCnt = 0
3097         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3098
3099         for info in p4CmdList(["files"] + fileArgs):
3100
3101             if 'code' in info and info['code'] == 'error':
3102                 sys.stderr.write("p4 returned an error: %s\n"
3103                                  % info['data'])
3104                 if info['data'].find("must refer to client") >= 0:
3105                     sys.stderr.write("This particular p4 error is misleading.\n")
3106                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3107                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3108                 sys.exit(1)
3109             if 'p4ExitCode' in info:
3110                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3111                 sys.exit(1)
3112
3113
3114             change = int(info["change"])
3115             if change > newestRevision:
3116                 newestRevision = change
3117
3118             if info["action"] in self.delete_actions:
3119                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3120                 #fileCnt = fileCnt + 1
3121                 continue
3122
3123             for prop in ["depotFile", "rev", "action", "type" ]:
3124                 details["%s%s" % (prop, fileCnt)] = info[prop]
3125
3126             fileCnt = fileCnt + 1
3127
3128         details["change"] = newestRevision
3129
3130         # Use time from top-most change so that all git p4 clones of
3131         # the same p4 repo have the same commit SHA1s.
3132         res = p4_describe(newestRevision)
3133         details["time"] = res["time"]
3134
3135         self.updateOptionDict(details)
3136         try:
3137             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3138         except IOError:
3139             print "IO error with git fast-import. Is your git version recent enough?"
3140             print self.gitError.read()
3141
3142
3143     def run(self, args):
3144         self.depotPaths = []
3145         self.changeRange = ""
3146         self.previousDepotPaths = []
3147         self.hasOrigin = False
3148
3149         # map from branch depot path to parent branch
3150         self.knownBranches = {}
3151         self.initialParents = {}
3152
3153         if self.importIntoRemotes:
3154             self.refPrefix = "refs/remotes/p4/"
3155         else:
3156             self.refPrefix = "refs/heads/p4/"
3157
3158         if self.syncWithOrigin:
3159             self.hasOrigin = originP4BranchesExist()
3160             if self.hasOrigin:
3161                 if not self.silent:
3162                     print 'Syncing with origin first, using "git fetch origin"'
3163                 system("git fetch origin")
3164
3165         branch_arg_given = bool(self.branch)
3166         if len(self.branch) == 0:
3167             self.branch = self.refPrefix + "master"
3168             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3169                 system("git update-ref %s refs/heads/p4" % self.branch)
3170                 system("git branch -D p4")
3171
3172         # accept either the command-line option, or the configuration variable
3173         if self.useClientSpec:
3174             # will use this after clone to set the variable
3175             self.useClientSpec_from_options = True
3176         else:
3177             if gitConfigBool("git-p4.useclientspec"):
3178                 self.useClientSpec = True
3179         if self.useClientSpec:
3180             self.clientSpecDirs = getClientSpec()
3181
3182         # TODO: should always look at previous commits,
3183         # merge with previous imports, if possible.
3184         if args == []:
3185             if self.hasOrigin:
3186                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3187
3188             # branches holds mapping from branch name to sha1
3189             branches = p4BranchesInGit(self.importIntoRemotes)
3190
3191             # restrict to just this one, disabling detect-branches
3192             if branch_arg_given:
3193                 short = self.branch.split("/")[-1]
3194                 if short in branches:
3195                     self.p4BranchesInGit = [ short ]
3196             else:
3197                 self.p4BranchesInGit = branches.keys()
3198
3199             if len(self.p4BranchesInGit) > 1:
3200                 if not self.silent:
3201                     print "Importing from/into multiple branches"
3202                 self.detectBranches = True
3203                 for branch in branches.keys():
3204                     self.initialParents[self.refPrefix + branch] = \
3205                         branches[branch]
3206
3207             if self.verbose:
3208                 print "branches: %s" % self.p4BranchesInGit
3209
3210             p4Change = 0
3211             for branch in self.p4BranchesInGit:
3212                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3213
3214                 settings = extractSettingsGitLog(logMsg)
3215
3216                 self.readOptions(settings)
3217                 if (settings.has_key('depot-paths')
3218                     and settings.has_key ('change')):
3219                     change = int(settings['change']) + 1
3220                     p4Change = max(p4Change, change)
3221
3222                     depotPaths = sorted(settings['depot-paths'])
3223                     if self.previousDepotPaths == []:
3224                         self.previousDepotPaths = depotPaths
3225                     else:
3226                         paths = []
3227                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3228                             prev_list = prev.split("/")
3229                             cur_list = cur.split("/")
3230                             for i in range(0, min(len(cur_list), len(prev_list))):
3231                                 if cur_list[i] <> prev_list[i]:
3232                                     i = i - 1
3233                                     break
3234
3235                             paths.append ("/".join(cur_list[:i + 1]))
3236
3237                         self.previousDepotPaths = paths
3238
3239             if p4Change > 0:
3240                 self.depotPaths = sorted(self.previousDepotPaths)
3241                 self.changeRange = "@%s,#head" % p4Change
3242                 if not self.silent and not self.detectBranches:
3243                     print "Performing incremental import into %s git branch" % self.branch
3244
3245         # accept multiple ref name abbreviations:
3246         #    refs/foo/bar/branch -> use it exactly
3247         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3248         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3249         if not self.branch.startswith("refs/"):
3250             if self.importIntoRemotes:
3251                 prepend = "refs/remotes/"
3252             else:
3253                 prepend = "refs/heads/"
3254             if not self.branch.startswith("p4/"):
3255                 prepend += "p4/"
3256             self.branch = prepend + self.branch
3257
3258         if len(args) == 0 and self.depotPaths:
3259             if not self.silent:
3260                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3261         else:
3262             if self.depotPaths and self.depotPaths != args:
3263                 print ("previous import used depot path %s and now %s was specified. "
3264                        "This doesn't work!" % (' '.join (self.depotPaths),
3265                                                ' '.join (args)))
3266                 sys.exit(1)
3267
3268             self.depotPaths = sorted(args)
3269
3270         revision = ""
3271         self.users = {}
3272
3273         # Make sure no revision specifiers are used when --changesfile
3274         # is specified.
3275         bad_changesfile = False
3276         if len(self.changesFile) > 0:
3277             for p in self.depotPaths:
3278                 if p.find("@") >= 0 or p.find("#") >= 0:
3279                     bad_changesfile = True
3280                     break
3281         if bad_changesfile:
3282             die("Option --changesfile is incompatible with revision specifiers")
3283
3284         newPaths = []
3285         for p in self.depotPaths:
3286             if p.find("@") != -1:
3287                 atIdx = p.index("@")
3288                 self.changeRange = p[atIdx:]
3289                 if self.changeRange == "@all":
3290                     self.changeRange = ""
3291                 elif ',' not in self.changeRange:
3292                     revision = self.changeRange
3293                     self.changeRange = ""
3294                 p = p[:atIdx]
3295             elif p.find("#") != -1:
3296                 hashIdx = p.index("#")
3297                 revision = p[hashIdx:]
3298                 p = p[:hashIdx]
3299             elif self.previousDepotPaths == []:
3300                 # pay attention to changesfile, if given, else import
3301                 # the entire p4 tree at the head revision
3302                 if len(self.changesFile) == 0:
3303                     revision = "#head"
3304
3305             p = re.sub ("\.\.\.$", "", p)
3306             if not p.endswith("/"):
3307                 p += "/"
3308
3309             newPaths.append(p)
3310
3311         self.depotPaths = newPaths
3312
3313         # --detect-branches may change this for each branch
3314         self.branchPrefixes = self.depotPaths
3315
3316         self.loadUserMapFromCache()
3317         self.labels = {}
3318         if self.detectLabels:
3319             self.getLabels();
3320
3321         if self.detectBranches:
3322             ## FIXME - what's a P4 projectName ?
3323             self.projectName = self.guessProjectName()
3324
3325             if self.hasOrigin:
3326                 self.getBranchMappingFromGitBranches()
3327             else:
3328                 self.getBranchMapping()
3329             if self.verbose:
3330                 print "p4-git branches: %s" % self.p4BranchesInGit
3331                 print "initial parents: %s" % self.initialParents
3332             for b in self.p4BranchesInGit:
3333                 if b != "master":
3334
3335                     ## FIXME
3336                     b = b[len(self.projectName):]
3337                 self.createdBranches.add(b)
3338
3339         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3340
3341         self.importProcess = subprocess.Popen(["git", "fast-import"],
3342                                               stdin=subprocess.PIPE,
3343                                               stdout=subprocess.PIPE,
3344                                               stderr=subprocess.PIPE);
3345         self.gitOutput = self.importProcess.stdout
3346         self.gitStream = self.importProcess.stdin
3347         self.gitError = self.importProcess.stderr
3348
3349         if revision:
3350             self.importHeadRevision(revision)
3351         else:
3352             changes = []
3353
3354             if len(self.changesFile) > 0:
3355                 output = open(self.changesFile).readlines()
3356                 changeSet = set()
3357                 for line in output:
3358                     changeSet.add(int(line))
3359
3360                 for change in changeSet:
3361                     changes.append(change)
3362
3363                 changes.sort()
3364             else:
3365                 # catch "git p4 sync" with no new branches, in a repo that
3366                 # does not have any existing p4 branches
3367                 if len(args) == 0:
3368                     if not self.p4BranchesInGit:
3369                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3370
3371                     # The default branch is master, unless --branch is used to
3372                     # specify something else.  Make sure it exists, or complain
3373                     # nicely about how to use --branch.
3374                     if not self.detectBranches:
3375                         if not branch_exists(self.branch):
3376                             if branch_arg_given:
3377                                 die("Error: branch %s does not exist." % self.branch)
3378                             else:
3379                                 die("Error: no branch %s; perhaps specify one with --branch." %
3380                                     self.branch)
3381
3382                 if self.verbose:
3383                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3384                                                               self.changeRange)
3385                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3386
3387                 if len(self.maxChanges) > 0:
3388                     changes = changes[:min(int(self.maxChanges), len(changes))]
3389
3390             if len(changes) == 0:
3391                 if not self.silent:
3392                     print "No changes to import!"
3393             else:
3394                 if not self.silent and not self.detectBranches:
3395                     print "Import destination: %s" % self.branch
3396
3397                 self.updatedBranches = set()
3398
3399                 if not self.detectBranches:
3400                     if args:
3401                         # start a new branch
3402                         self.initialParent = ""
3403                     else:
3404                         # build on a previous revision
3405                         self.initialParent = parseRevision(self.branch)
3406
3407                 self.importChanges(changes)
3408
3409                 if not self.silent:
3410                     print ""
3411                     if len(self.updatedBranches) > 0:
3412                         sys.stdout.write("Updated branches: ")
3413                         for b in self.updatedBranches:
3414                             sys.stdout.write("%s " % b)
3415                         sys.stdout.write("\n")
3416
3417         if gitConfigBool("git-p4.importLabels"):
3418             self.importLabels = True
3419
3420         if self.importLabels:
3421             p4Labels = getP4Labels(self.depotPaths)
3422             gitTags = getGitTags()
3423
3424             missingP4Labels = p4Labels - gitTags
3425             self.importP4Labels(self.gitStream, missingP4Labels)
3426
3427         self.gitStream.close()
3428         if self.importProcess.wait() != 0:
3429             die("fast-import failed: %s" % self.gitError.read())
3430         self.gitOutput.close()
3431         self.gitError.close()
3432
3433         # Cleanup temporary branches created during import
3434         if self.tempBranches != []:
3435             for branch in self.tempBranches:
3436                 read_pipe("git update-ref -d %s" % branch)
3437             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3438
3439         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3440         # a convenient shortcut refname "p4".
3441         if self.importIntoRemotes:
3442             head_ref = self.refPrefix + "HEAD"
3443             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3444                 system(["git", "symbolic-ref", head_ref, self.branch])
3445
3446         return True
3447
3448 class P4Rebase(Command):
3449     def __init__(self):
3450         Command.__init__(self)
3451         self.options = [
3452                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3453         ]
3454         self.importLabels = False
3455         self.description = ("Fetches the latest revision from perforce and "
3456                             + "rebases the current work (branch) against it")
3457
3458     def run(self, args):
3459         sync = P4Sync()
3460         sync.importLabels = self.importLabels
3461         sync.run([])
3462
3463         return self.rebase()
3464
3465     def rebase(self):
3466         if os.system("git update-index --refresh") != 0:
3467             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3468         if len(read_pipe("git diff-index HEAD --")) > 0:
3469             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3470
3471         [upstream, settings] = findUpstreamBranchPoint()
3472         if len(upstream) == 0:
3473             die("Cannot find upstream branchpoint for rebase")
3474
3475         # the branchpoint may be p4/foo~3, so strip off the parent
3476         upstream = re.sub("~[0-9]+$", "", upstream)
3477
3478         print "Rebasing the current branch onto %s" % upstream
3479         oldHead = read_pipe("git rev-parse HEAD").strip()
3480         system("git rebase %s" % upstream)
3481         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3482         return True
3483
3484 class P4Clone(P4Sync):
3485     def __init__(self):
3486         P4Sync.__init__(self)
3487         self.description = "Creates a new git repository and imports from Perforce into it"
3488         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3489         self.options += [
3490             optparse.make_option("--destination", dest="cloneDestination",
3491                                  action='store', default=None,
3492                                  help="where to leave result of the clone"),
3493             optparse.make_option("--bare", dest="cloneBare",
3494                                  action="store_true", default=False),
3495         ]
3496         self.cloneDestination = None
3497         self.needsGit = False
3498         self.cloneBare = False
3499
3500     def defaultDestination(self, args):
3501         ## TODO: use common prefix of args?
3502         depotPath = args[0]
3503         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3504         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3505         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3506         depotDir = re.sub(r"/$", "", depotDir)
3507         return os.path.split(depotDir)[1]
3508
3509     def run(self, args):
3510         if len(args) < 1:
3511             return False
3512
3513         if self.keepRepoPath and not self.cloneDestination:
3514             sys.stderr.write("Must specify destination for --keep-path\n")
3515             sys.exit(1)
3516
3517         depotPaths = args
3518
3519         if not self.cloneDestination and len(depotPaths) > 1:
3520             self.cloneDestination = depotPaths[-1]
3521             depotPaths = depotPaths[:-1]
3522
3523         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3524         for p in depotPaths:
3525             if not p.startswith("//"):
3526                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3527                 return False
3528
3529         if not self.cloneDestination:
3530             self.cloneDestination = self.defaultDestination(args)
3531
3532         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3533
3534         if not os.path.exists(self.cloneDestination):
3535             os.makedirs(self.cloneDestination)
3536         chdir(self.cloneDestination)
3537
3538         init_cmd = [ "git", "init" ]
3539         if self.cloneBare:
3540             init_cmd.append("--bare")
3541         retcode = subprocess.call(init_cmd)
3542         if retcode:
3543             raise CalledProcessError(retcode, init_cmd)
3544
3545         if not P4Sync.run(self, depotPaths):
3546             return False
3547
3548         # create a master branch and check out a work tree
3549         if gitBranchExists(self.branch):
3550             system([ "git", "branch", "master", self.branch ])
3551             if not self.cloneBare:
3552                 system([ "git", "checkout", "-f" ])
3553         else:
3554             print 'Not checking out any branch, use ' \
3555                   '"git checkout -q -b master <branch>"'
3556
3557         # auto-set this variable if invoked with --use-client-spec
3558         if self.useClientSpec_from_options:
3559             system("git config --bool git-p4.useclientspec true")
3560
3561         return True
3562
3563 class P4Branches(Command):
3564     def __init__(self):
3565         Command.__init__(self)
3566         self.options = [ ]
3567         self.description = ("Shows the git branches that hold imports and their "
3568                             + "corresponding perforce depot paths")
3569         self.verbose = False
3570
3571     def run(self, args):
3572         if originP4BranchesExist():
3573             createOrUpdateBranchesFromOrigin()
3574
3575         cmdline = "git rev-parse --symbolic "
3576         cmdline += " --remotes"
3577
3578         for line in read_pipe_lines(cmdline):
3579             line = line.strip()
3580
3581             if not line.startswith('p4/') or line == "p4/HEAD":
3582                 continue
3583             branch = line
3584
3585             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3586             settings = extractSettingsGitLog(log)
3587
3588             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3589         return True
3590
3591 class HelpFormatter(optparse.IndentedHelpFormatter):
3592     def __init__(self):
3593         optparse.IndentedHelpFormatter.__init__(self)
3594
3595     def format_description(self, description):
3596         if description:
3597             return description + "\n"
3598         else:
3599             return ""
3600
3601 def printUsage(commands):
3602     print "usage: %s <command> [options]" % sys.argv[0]
3603     print ""
3604     print "valid commands: %s" % ", ".join(commands)
3605     print ""
3606     print "Try %s <command> --help for command specific help." % sys.argv[0]
3607     print ""
3608
3609 commands = {
3610     "debug" : P4Debug,
3611     "submit" : P4Submit,
3612     "commit" : P4Submit,
3613     "sync" : P4Sync,
3614     "rebase" : P4Rebase,
3615     "clone" : P4Clone,
3616     "rollback" : P4RollBack,
3617     "branches" : P4Branches
3618 }
3619
3620
3621 def main():
3622     if len(sys.argv[1:]) == 0:
3623         printUsage(commands.keys())
3624         sys.exit(2)
3625
3626     cmdName = sys.argv[1]
3627     try:
3628         klass = commands[cmdName]
3629         cmd = klass()
3630     except KeyError:
3631         print "unknown command %s" % cmdName
3632         print ""
3633         printUsage(commands.keys())
3634         sys.exit(2)
3635
3636     options = cmd.options
3637     cmd.gitdir = os.environ.get("GIT_DIR", None)
3638
3639     args = sys.argv[2:]
3640
3641     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3642     if cmd.needsGit:
3643         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3644
3645     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3646                                    options,
3647                                    description = cmd.description,
3648                                    formatter = HelpFormatter())
3649
3650     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3651     global verbose
3652     verbose = cmd.verbose
3653     if cmd.needsGit:
3654         if cmd.gitdir == None:
3655             cmd.gitdir = os.path.abspath(".git")
3656             if not isValidGitDir(cmd.gitdir):
3657                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3658                 if os.path.exists(cmd.gitdir):
3659                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3660                     if len(cdup) > 0:
3661                         chdir(cdup);
3662
3663         if not isValidGitDir(cmd.gitdir):
3664             if isValidGitDir(cmd.gitdir + "/.git"):
3665                 cmd.gitdir += "/.git"
3666             else:
3667                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3668
3669         os.environ["GIT_DIR"] = cmd.gitdir
3670
3671     if not cmd.run(args):
3672         parser.print_help()
3673         sys.exit(2)
3674
3675
3676 if __name__ == '__main__':
3677     main()