Merge branch 'sb/clone-shallow-passthru'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28
29 try:
30     from subprocess import CalledProcessError
31 except ImportError:
32     # from python2.7:subprocess.py
33     # Exception classes used by this module.
34     class CalledProcessError(Exception):
35         """This exception is raised when a process run by check_call() returns
36         a non-zero exit status.  The exit status will be stored in the
37         returncode attribute."""
38         def __init__(self, returncode, cmd):
39             self.returncode = returncode
40             self.cmd = cmd
41         def __str__(self):
42             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
44 verbose = False
45
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
48
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
51
52 def p4_build_cmd(cmd):
53     """Build a suitable p4 command line.
54
55     This consolidates building and returning a p4 command line into one
56     location. It means that hooking into the environment, or other configuration
57     can be done more easily.
58     """
59     real_cmd = ["p4"]
60
61     user = gitConfig("git-p4.user")
62     if len(user) > 0:
63         real_cmd += ["-u",user]
64
65     password = gitConfig("git-p4.password")
66     if len(password) > 0:
67         real_cmd += ["-P", password]
68
69     port = gitConfig("git-p4.port")
70     if len(port) > 0:
71         real_cmd += ["-p", port]
72
73     host = gitConfig("git-p4.host")
74     if len(host) > 0:
75         real_cmd += ["-H", host]
76
77     client = gitConfig("git-p4.client")
78     if len(client) > 0:
79         real_cmd += ["-c", client]
80
81
82     if isinstance(cmd,basestring):
83         real_cmd = ' '.join(real_cmd) + ' ' + cmd
84     else:
85         real_cmd += cmd
86     return real_cmd
87
88 def chdir(path, is_client_path=False):
89     """Do chdir to the given path, and set the PWD environment
90        variable for use by P4.  It does not look at getcwd() output.
91        Since we're not using the shell, it is necessary to set the
92        PWD environment variable explicitly.
93
94        Normally, expand the path to force it to be absolute.  This
95        addresses the use of relative path names inside P4 settings,
96        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
97        as given; it looks for .p4config using PWD.
98
99        If is_client_path, the path was handed to us directly by p4,
100        and may be a symbolic link.  Do not call os.getcwd() in this
101        case, because it will cause p4 to think that PWD is not inside
102        the client path.
103        """
104
105     os.chdir(path)
106     if not is_client_path:
107         path = os.getcwd()
108     os.environ['PWD'] = path
109
110 def calcDiskFree():
111     """Return free space in bytes on the disk of the given dirname."""
112     if platform.system() == 'Windows':
113         free_bytes = ctypes.c_ulonglong(0)
114         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115         return free_bytes.value
116     else:
117         st = os.statvfs(os.getcwd())
118         return st.f_bavail * st.f_frsize
119
120 def die(msg):
121     if verbose:
122         raise Exception(msg)
123     else:
124         sys.stderr.write(msg + "\n")
125         sys.exit(1)
126
127 def write_pipe(c, stdin):
128     if verbose:
129         sys.stderr.write('Writing pipe: %s\n' % str(c))
130
131     expand = isinstance(c,basestring)
132     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133     pipe = p.stdin
134     val = pipe.write(stdin)
135     pipe.close()
136     if p.wait():
137         die('Command failed: %s' % str(c))
138
139     return val
140
141 def p4_write_pipe(c, stdin):
142     real_cmd = p4_build_cmd(c)
143     return write_pipe(real_cmd, stdin)
144
145 def read_pipe(c, ignore_error=False):
146     if verbose:
147         sys.stderr.write('Reading pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151     (out, err) = p.communicate()
152     if p.returncode != 0 and not ignore_error:
153         die('Command failed: %s\nError: %s' % (str(c), err))
154     return out
155
156 def p4_read_pipe(c, ignore_error=False):
157     real_cmd = p4_build_cmd(c)
158     return read_pipe(real_cmd, ignore_error)
159
160 def read_pipe_lines(c):
161     if verbose:
162         sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164     expand = isinstance(c, basestring)
165     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166     pipe = p.stdout
167     val = pipe.readlines()
168     if pipe.close() or p.wait():
169         die('Command failed: %s' % str(c))
170
171     return val
172
173 def p4_read_pipe_lines(c):
174     """Specifically invoke p4 on the command supplied. """
175     real_cmd = p4_build_cmd(c)
176     return read_pipe_lines(real_cmd)
177
178 def p4_has_command(cmd):
179     """Ask p4 for help on this command.  If it returns an error, the
180        command does not exist in this version of p4."""
181     real_cmd = p4_build_cmd(["help", cmd])
182     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183                                    stderr=subprocess.PIPE)
184     p.communicate()
185     return p.returncode == 0
186
187 def p4_has_move_command():
188     """See if the move command exists, that it supports -k, and that
189        it has not been administratively disabled.  The arguments
190        must be correct, but the filenames do not have to exist.  Use
191        ones with wildcards so even if they exist, it will fail."""
192
193     if not p4_has_command("move"):
194         return False
195     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197     (out, err) = p.communicate()
198     # return code will be 1 in either case
199     if err.find("Invalid option") >= 0:
200         return False
201     if err.find("disabled") >= 0:
202         return False
203     # assume it failed because @... was invalid changelist
204     return True
205
206 def system(cmd, ignore_error=False):
207     expand = isinstance(cmd,basestring)
208     if verbose:
209         sys.stderr.write("executing %s\n" % str(cmd))
210     retcode = subprocess.call(cmd, shell=expand)
211     if retcode and not ignore_error:
212         raise CalledProcessError(retcode, cmd)
213
214     return retcode
215
216 def p4_system(cmd):
217     """Specifically invoke p4 as the system command. """
218     real_cmd = p4_build_cmd(cmd)
219     expand = isinstance(real_cmd, basestring)
220     retcode = subprocess.call(real_cmd, shell=expand)
221     if retcode:
222         raise CalledProcessError(retcode, real_cmd)
223
224 _p4_version_string = None
225 def p4_version_string():
226     """Read the version string, showing just the last line, which
227        hopefully is the interesting version bit.
228
229        $ p4 -V
230        Perforce - The Fast Software Configuration Management System.
231        Copyright 1995-2011 Perforce Software.  All rights reserved.
232        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233     """
234     global _p4_version_string
235     if not _p4_version_string:
236         a = p4_read_pipe_lines(["-V"])
237         _p4_version_string = a[-1].rstrip()
238     return _p4_version_string
239
240 def p4_integrate(src, dest):
241     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
242
243 def p4_sync(f, *options):
244     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
245
246 def p4_add(f):
247     # forcibly add file names with wildcards
248     if wildcard_present(f):
249         p4_system(["add", "-f", f])
250     else:
251         p4_system(["add", f])
252
253 def p4_delete(f):
254     p4_system(["delete", wildcard_encode(f)])
255
256 def p4_edit(f, *options):
257     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
258
259 def p4_revert(f):
260     p4_system(["revert", wildcard_encode(f)])
261
262 def p4_reopen(type, f):
263     p4_system(["reopen", "-t", type, wildcard_encode(f)])
264
265 def p4_move(src, dest):
266     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
267
268 def p4_last_change():
269     results = p4CmdList(["changes", "-m", "1"])
270     return int(results[0]['change'])
271
272 def p4_describe(change):
273     """Make sure it returns a valid result by checking for
274        the presence of field "time".  Return a dict of the
275        results."""
276
277     ds = p4CmdList(["describe", "-s", str(change)])
278     if len(ds) != 1:
279         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280
281     d = ds[0]
282
283     if "p4ExitCode" in d:
284         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
285                                                       str(d)))
286     if "code" in d:
287         if d["code"] == "error":
288             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
289
290     if "time" not in d:
291         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
292
293     return d
294
295 #
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers.  See "p4 help filetypes"
298 # for a list and explanation.
299 #
300 def split_p4_type(p4type):
301
302     p4_filetypes_historical = {
303         "ctempobj": "binary+Sw",
304         "ctext": "text+C",
305         "cxtext": "text+Cx",
306         "ktext": "text+k",
307         "kxtext": "text+kx",
308         "ltext": "text+F",
309         "tempobj": "binary+FSw",
310         "ubinary": "binary+F",
311         "uresource": "resource+F",
312         "uxbinary": "binary+Fx",
313         "xbinary": "binary+x",
314         "xltext": "text+Fx",
315         "xtempobj": "binary+Swx",
316         "xtext": "text+x",
317         "xunicode": "unicode+x",
318         "xutf16": "utf16+x",
319     }
320     if p4type in p4_filetypes_historical:
321         p4type = p4_filetypes_historical[p4type]
322     mods = ""
323     s = p4type.split("+")
324     base = s[0]
325     mods = ""
326     if len(s) > 1:
327         mods = s[1]
328     return (base, mods)
329
330 #
331 # return the raw p4 type of a file (text, text+ko, etc)
332 #
333 def p4_type(f):
334     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
335     return results[0]['headType']
336
337 #
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
340 #
341 def p4_keywords_regexp_for_type(base, type_mods):
342     if base in ("text", "unicode", "binary"):
343         kwords = None
344         if "ko" in type_mods:
345             kwords = 'Id|Header'
346         elif "k" in type_mods:
347             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348         else:
349             return None
350         pattern = r"""
351             \$              # Starts with a dollar, followed by...
352             (%s)            # one of the keywords, followed by...
353             (:[^$\n]+)?     # possibly an old expansion, followed by...
354             \$              # another dollar
355             """ % kwords
356         return pattern
357     else:
358         return None
359
360 #
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
364 #
365 def p4_keywords_regexp_for_file(file):
366     if not os.path.exists(file):
367         return None
368     else:
369         (type_base, type_mods) = split_p4_type(p4_type(file))
370         return p4_keywords_regexp_for_type(type_base, type_mods)
371
372 def setP4ExecBit(file, mode):
373     # Reopens an already open file and changes the execute bit to match
374     # the execute bit setting in the passed in mode.
375
376     p4Type = "+x"
377
378     if not isModeExec(mode):
379         p4Type = getP4OpenedType(file)
380         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382         if p4Type[-1] == "+":
383             p4Type = p4Type[0:-1]
384
385     p4_reopen(p4Type, file)
386
387 def getP4OpenedType(file):
388     # Returns the perforce file type for the given file.
389
390     result = p4_read_pipe(["opened", wildcard_encode(file)])
391     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
392     if match:
393         return match.group(1)
394     else:
395         die("Could not determine file type for %s (result: '%s')" % (file, result))
396
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths):
399     labels = set()
400     if isinstance(depotPaths,basestring):
401         depotPaths = [depotPaths]
402
403     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
404         label = l['label']
405         labels.add(label)
406
407     return labels
408
409 # Return the set of all git tags
410 def getGitTags():
411     gitTags = set()
412     for line in read_pipe_lines(["git", "tag"]):
413         tag = line.strip()
414         gitTags.add(tag)
415     return gitTags
416
417 def diffTreePattern():
418     # This is a simple generator for the diff tree regex pattern. This could be
419     # a class variable if this and parseDiffTreeEntry were a part of a class.
420     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421     while True:
422         yield pattern
423
424 def parseDiffTreeEntry(entry):
425     """Parses a single diff tree entry into its component elements.
426
427     See git-diff-tree(1) manpage for details about the format of the diff
428     output. This method returns a dictionary with the following elements:
429
430     src_mode - The mode of the source file
431     dst_mode - The mode of the destination file
432     src_sha1 - The sha1 for the source file
433     dst_sha1 - The sha1 fr the destination file
434     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435     status_score - The score for the status (applicable for 'C' and 'R'
436                    statuses). This is None if there is no score.
437     src - The path for the source file.
438     dst - The path for the destination file. This is only present for
439           copy or renames. If it is not present, this is None.
440
441     If the pattern is not matched, None is returned."""
442
443     match = diffTreePattern().next().match(entry)
444     if match:
445         return {
446             'src_mode': match.group(1),
447             'dst_mode': match.group(2),
448             'src_sha1': match.group(3),
449             'dst_sha1': match.group(4),
450             'status': match.group(5),
451             'status_score': match.group(6),
452             'src': match.group(7),
453             'dst': match.group(10)
454         }
455     return None
456
457 def isModeExec(mode):
458     # Returns True if the given git mode represents an executable file,
459     # otherwise False.
460     return mode[-3:] == "755"
461
462 def isModeExecChanged(src_mode, dst_mode):
463     return isModeExec(src_mode) != isModeExec(dst_mode)
464
465 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
466
467     if isinstance(cmd,basestring):
468         cmd = "-G " + cmd
469         expand = True
470     else:
471         cmd = ["-G"] + cmd
472         expand = False
473
474     cmd = p4_build_cmd(cmd)
475     if verbose:
476         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
477
478     # Use a temporary file to avoid deadlocks without
479     # subprocess.communicate(), which would put another copy
480     # of stdout into memory.
481     stdin_file = None
482     if stdin is not None:
483         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
484         if isinstance(stdin,basestring):
485             stdin_file.write(stdin)
486         else:
487             for i in stdin:
488                 stdin_file.write(i + '\n')
489         stdin_file.flush()
490         stdin_file.seek(0)
491
492     p4 = subprocess.Popen(cmd,
493                           shell=expand,
494                           stdin=stdin_file,
495                           stdout=subprocess.PIPE)
496
497     result = []
498     try:
499         while True:
500             entry = marshal.load(p4.stdout)
501             if cb is not None:
502                 cb(entry)
503             else:
504                 result.append(entry)
505     except EOFError:
506         pass
507     exitCode = p4.wait()
508     if exitCode != 0:
509         entry = {}
510         entry["p4ExitCode"] = exitCode
511         result.append(entry)
512
513     return result
514
515 def p4Cmd(cmd):
516     list = p4CmdList(cmd)
517     result = {}
518     for entry in list:
519         result.update(entry)
520     return result;
521
522 def p4Where(depotPath):
523     if not depotPath.endswith("/"):
524         depotPath += "/"
525     depotPathLong = depotPath + "..."
526     outputList = p4CmdList(["where", depotPathLong])
527     output = None
528     for entry in outputList:
529         if "depotFile" in entry:
530             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531             # The base path always ends with "/...".
532             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
533                 output = entry
534                 break
535         elif "data" in entry:
536             data = entry.get("data")
537             space = data.find(" ")
538             if data[:space] == depotPath:
539                 output = entry
540                 break
541     if output == None:
542         return ""
543     if output["code"] == "error":
544         return ""
545     clientPath = ""
546     if "path" in output:
547         clientPath = output.get("path")
548     elif "data" in output:
549         data = output.get("data")
550         lastSpace = data.rfind(" ")
551         clientPath = data[lastSpace + 1:]
552
553     if clientPath.endswith("..."):
554         clientPath = clientPath[:-3]
555     return clientPath
556
557 def currentGitBranch():
558     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
559     if retcode != 0:
560         # on a detached head
561         return None
562     else:
563         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
564
565 def isValidGitDir(path):
566     if (os.path.exists(path + "/HEAD")
567         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
568         return True;
569     return False
570
571 def parseRevision(ref):
572     return read_pipe("git rev-parse %s" % ref).strip()
573
574 def branchExists(ref):
575     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
576                      ignore_error=True)
577     return len(rev) > 0
578
579 def extractLogMessageFromGitCommit(commit):
580     logMessage = ""
581
582     ## fixme: title is first line of commit, not 1st paragraph.
583     foundTitle = False
584     for log in read_pipe_lines("git cat-file commit %s" % commit):
585        if not foundTitle:
586            if len(log) == 1:
587                foundTitle = True
588            continue
589
590        logMessage += log
591     return logMessage
592
593 def extractSettingsGitLog(log):
594     values = {}
595     for line in log.split("\n"):
596         line = line.strip()
597         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
598         if not m:
599             continue
600
601         assignments = m.group(1).split (':')
602         for a in assignments:
603             vals = a.split ('=')
604             key = vals[0].strip()
605             val = ('='.join (vals[1:])).strip()
606             if val.endswith ('\"') and val.startswith('"'):
607                 val = val[1:-1]
608
609             values[key] = val
610
611     paths = values.get("depot-paths")
612     if not paths:
613         paths = values.get("depot-path")
614     if paths:
615         values['depot-paths'] = paths.split(',')
616     return values
617
618 def gitBranchExists(branch):
619     proc = subprocess.Popen(["git", "rev-parse", branch],
620                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
621     return proc.wait() == 0;
622
623 _gitConfig = {}
624
625 def gitConfig(key, typeSpecifier=None):
626     if not _gitConfig.has_key(key):
627         cmd = [ "git", "config" ]
628         if typeSpecifier:
629             cmd += [ typeSpecifier ]
630         cmd += [ key ]
631         s = read_pipe(cmd, ignore_error=True)
632         _gitConfig[key] = s.strip()
633     return _gitConfig[key]
634
635 def gitConfigBool(key):
636     """Return a bool, using git config --bool.  It is True only if the
637        variable is set to true, and False if set to false or not present
638        in the config."""
639
640     if not _gitConfig.has_key(key):
641         _gitConfig[key] = gitConfig(key, '--bool') == "true"
642     return _gitConfig[key]
643
644 def gitConfigInt(key):
645     if not _gitConfig.has_key(key):
646         cmd = [ "git", "config", "--int", key ]
647         s = read_pipe(cmd, ignore_error=True)
648         v = s.strip()
649         try:
650             _gitConfig[key] = int(gitConfig(key, '--int'))
651         except ValueError:
652             _gitConfig[key] = None
653     return _gitConfig[key]
654
655 def gitConfigList(key):
656     if not _gitConfig.has_key(key):
657         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658         _gitConfig[key] = s.strip().split(os.linesep)
659         if _gitConfig[key] == ['']:
660             _gitConfig[key] = []
661     return _gitConfig[key]
662
663 def p4BranchesInGit(branchesAreInRemotes=True):
664     """Find all the branches whose names start with "p4/", looking
665        in remotes or heads as specified by the argument.  Return
666        a dictionary of { branch: revision } for each one found.
667        The branch names are the short names, without any
668        "p4/" prefix."""
669
670     branches = {}
671
672     cmdline = "git rev-parse --symbolic "
673     if branchesAreInRemotes:
674         cmdline += "--remotes"
675     else:
676         cmdline += "--branches"
677
678     for line in read_pipe_lines(cmdline):
679         line = line.strip()
680
681         # only import to p4/
682         if not line.startswith('p4/'):
683             continue
684         # special symbolic ref to p4/master
685         if line == "p4/HEAD":
686             continue
687
688         # strip off p4/ prefix
689         branch = line[len("p4/"):]
690
691         branches[branch] = parseRevision(line)
692
693     return branches
694
695 def branch_exists(branch):
696     """Make sure that the given ref name really exists."""
697
698     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700     out, _ = p.communicate()
701     if p.returncode:
702         return False
703     # expect exactly one line of output: the branch name
704     return out.rstrip() == branch
705
706 def findUpstreamBranchPoint(head = "HEAD"):
707     branches = p4BranchesInGit()
708     # map from depot-path to branch name
709     branchByDepotPath = {}
710     for branch in branches.keys():
711         tip = branches[branch]
712         log = extractLogMessageFromGitCommit(tip)
713         settings = extractSettingsGitLog(log)
714         if settings.has_key("depot-paths"):
715             paths = ",".join(settings["depot-paths"])
716             branchByDepotPath[paths] = "remotes/p4/" + branch
717
718     settings = None
719     parent = 0
720     while parent < 65535:
721         commit = head + "~%s" % parent
722         log = extractLogMessageFromGitCommit(commit)
723         settings = extractSettingsGitLog(log)
724         if settings.has_key("depot-paths"):
725             paths = ",".join(settings["depot-paths"])
726             if branchByDepotPath.has_key(paths):
727                 return [branchByDepotPath[paths], settings]
728
729         parent = parent + 1
730
731     return ["", settings]
732
733 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
734     if not silent:
735         print ("Creating/updating branch(es) in %s based on origin branch(es)"
736                % localRefPrefix)
737
738     originPrefix = "origin/p4/"
739
740     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
741         line = line.strip()
742         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
743             continue
744
745         headName = line[len(originPrefix):]
746         remoteHead = localRefPrefix + headName
747         originHead = line
748
749         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750         if (not original.has_key('depot-paths')
751             or not original.has_key('change')):
752             continue
753
754         update = False
755         if not gitBranchExists(remoteHead):
756             if verbose:
757                 print "creating %s" % remoteHead
758             update = True
759         else:
760             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761             if settings.has_key('change') > 0:
762                 if settings['depot-paths'] == original['depot-paths']:
763                     originP4Change = int(original['change'])
764                     p4Change = int(settings['change'])
765                     if originP4Change > p4Change:
766                         print ("%s (%s) is newer than %s (%s). "
767                                "Updating p4 branch from origin."
768                                % (originHead, originP4Change,
769                                   remoteHead, p4Change))
770                         update = True
771                 else:
772                     print ("Ignoring: %s was imported from %s while "
773                            "%s was imported from %s"
774                            % (originHead, ','.join(original['depot-paths']),
775                               remoteHead, ','.join(settings['depot-paths'])))
776
777         if update:
778             system("git update-ref %s %s" % (remoteHead, originHead))
779
780 def originP4BranchesExist():
781         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
782
783
784 def p4ParseNumericChangeRange(parts):
785     changeStart = int(parts[0][1:])
786     if parts[1] == '#head':
787         changeEnd = p4_last_change()
788     else:
789         changeEnd = int(parts[1])
790
791     return (changeStart, changeEnd)
792
793 def chooseBlockSize(blockSize):
794     if blockSize:
795         return blockSize
796     else:
797         return defaultBlockSize
798
799 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
800     assert depotPaths
801
802     # Parse the change range into start and end. Try to find integer
803     # revision ranges as these can be broken up into blocks to avoid
804     # hitting server-side limits (maxrows, maxscanresults). But if
805     # that doesn't work, fall back to using the raw revision specifier
806     # strings, without using block mode.
807
808     if changeRange is None or changeRange == '':
809         changeStart = 1
810         changeEnd = p4_last_change()
811         block_size = chooseBlockSize(requestedBlockSize)
812     else:
813         parts = changeRange.split(',')
814         assert len(parts) == 2
815         try:
816             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817             block_size = chooseBlockSize(requestedBlockSize)
818         except:
819             changeStart = parts[0][1:]
820             changeEnd = parts[1]
821             if requestedBlockSize:
822                 die("cannot use --changes-block-size with non-numeric revisions")
823             block_size = None
824
825     changes = []
826
827     # Retrieve changes a block at a time, to prevent running
828     # into a MaxResults/MaxScanRows error from the server.
829
830     while True:
831         cmd = ['changes']
832
833         if block_size:
834             end = min(changeEnd, changeStart + block_size)
835             revisionRange = "%d,%d" % (changeStart, end)
836         else:
837             revisionRange = "%s,%s" % (changeStart, changeEnd)
838
839         for p in depotPaths:
840             cmd += ["%s...@%s" % (p, revisionRange)]
841
842         # Insert changes in chronological order
843         for line in reversed(p4_read_pipe_lines(cmd)):
844             changes.append(int(line.split(" ")[1]))
845
846         if not block_size:
847             break
848
849         if end >= changeEnd:
850             break
851
852         changeStart = end + 1
853
854     changes = sorted(changes)
855     return changes
856
857 def p4PathStartsWith(path, prefix):
858     # This method tries to remedy a potential mixed-case issue:
859     #
860     # If UserA adds  //depot/DirA/file1
861     # and UserB adds //depot/dira/file2
862     #
863     # we may or may not have a problem. If you have core.ignorecase=true,
864     # we treat DirA and dira as the same directory
865     if gitConfigBool("core.ignorecase"):
866         return path.lower().startswith(prefix.lower())
867     return path.startswith(prefix)
868
869 def getClientSpec():
870     """Look at the p4 client spec, create a View() object that contains
871        all the mappings, and return it."""
872
873     specList = p4CmdList("client -o")
874     if len(specList) != 1:
875         die('Output from "client -o" is %d lines, expecting 1' %
876             len(specList))
877
878     # dictionary of all client parameters
879     entry = specList[0]
880
881     # the //client/ name
882     client_name = entry["Client"]
883
884     # just the keys that start with "View"
885     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
886
887     # hold this new View
888     view = View(client_name)
889
890     # append the lines, in order, to the view
891     for view_num in range(len(view_keys)):
892         k = "View%d" % view_num
893         if k not in view_keys:
894             die("Expected view key %s missing" % k)
895         view.append(entry[k])
896
897     return view
898
899 def getClientRoot():
900     """Grab the client directory."""
901
902     output = p4CmdList("client -o")
903     if len(output) != 1:
904         die('Output from "client -o" is %d lines, expecting 1' % len(output))
905
906     entry = output[0]
907     if "Root" not in entry:
908         die('Client has no "Root"')
909
910     return entry["Root"]
911
912 #
913 # P4 wildcards are not allowed in filenames.  P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
916 #
917 def wildcard_decode(path):
918     # Search for and fix just these four characters.  Do % last so
919     # that fixing it does not inadvertently create new %-escapes.
920     # Cannot have * in a filename in windows; untested as to
921     # what p4 would do in such a case.
922     if not platform.system() == "Windows":
923         path = path.replace("%2A", "*")
924     path = path.replace("%23", "#") \
925                .replace("%40", "@") \
926                .replace("%25", "%")
927     return path
928
929 def wildcard_encode(path):
930     # do % first to avoid double-encoding the %s introduced here
931     path = path.replace("%", "%25") \
932                .replace("*", "%2A") \
933                .replace("#", "%23") \
934                .replace("@", "%40")
935     return path
936
937 def wildcard_present(path):
938     m = re.search("[*#@%]", path)
939     return m is not None
940
941 class LargeFileSystem(object):
942     """Base class for large file system support."""
943
944     def __init__(self, writeToGitStream):
945         self.largeFiles = set()
946         self.writeToGitStream = writeToGitStream
947
948     def generatePointer(self, cloneDestination, contentFile):
949         """Return the content of a pointer file that is stored in Git instead of
950            the actual content."""
951         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
952
953     def pushFile(self, localLargeFile):
954         """Push the actual content which is not stored in the Git repository to
955            a server."""
956         assert False, "Method 'pushFile' required in " + self.__class__.__name__
957
958     def hasLargeFileExtension(self, relPath):
959         return reduce(
960             lambda a, b: a or b,
961             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
962             False
963         )
964
965     def generateTempFile(self, contents):
966         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
967         for d in contents:
968             contentFile.write(d)
969         contentFile.close()
970         return contentFile.name
971
972     def exceedsLargeFileThreshold(self, relPath, contents):
973         if gitConfigInt('git-p4.largeFileThreshold'):
974             contentsSize = sum(len(d) for d in contents)
975             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
976                 return True
977         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978             contentsSize = sum(len(d) for d in contents)
979             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
980                 return False
981             contentTempFile = self.generateTempFile(contents)
982             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
985             zf.close()
986             compressedContentsSize = zf.infolist()[0].compress_size
987             os.remove(contentTempFile)
988             os.remove(compressedContentFile.name)
989             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
990                 return True
991         return False
992
993     def addLargeFile(self, relPath):
994         self.largeFiles.add(relPath)
995
996     def removeLargeFile(self, relPath):
997         self.largeFiles.remove(relPath)
998
999     def isLargeFile(self, relPath):
1000         return relPath in self.largeFiles
1001
1002     def processContent(self, git_mode, relPath, contents):
1003         """Processes the content of git fast import. This method decides if a
1004            file is stored in the large file system and handles all necessary
1005            steps."""
1006         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007             contentTempFile = self.generateTempFile(contents)
1008             (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1009
1010             # Move temp file to final location in large file system
1011             largeFileDir = os.path.dirname(localLargeFile)
1012             if not os.path.isdir(largeFileDir):
1013                 os.makedirs(largeFileDir)
1014             shutil.move(contentTempFile, localLargeFile)
1015             self.addLargeFile(relPath)
1016             if gitConfigBool('git-p4.largeFilePush'):
1017                 self.pushFile(localLargeFile)
1018             if verbose:
1019                 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020         return (git_mode, contents)
1021
1022 class MockLFS(LargeFileSystem):
1023     """Mock large file system for testing."""
1024
1025     def generatePointer(self, contentFile):
1026         """The pointer content is the original content prefixed with "pointer-".
1027            The local filename of the large file storage is derived from the file content.
1028            """
1029         with open(contentFile, 'r') as f:
1030             content = next(f)
1031             gitMode = '100644'
1032             pointerContents = 'pointer-' + content
1033             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034             return (gitMode, pointerContents, localLargeFile)
1035
1036     def pushFile(self, localLargeFile):
1037         """The remote filename of the large file storage is the same as the local
1038            one but in a different directory.
1039            """
1040         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041         if not os.path.exists(remotePath):
1042             os.makedirs(remotePath)
1043         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1044
1045 class GitLFS(LargeFileSystem):
1046     """Git LFS as backend for the git-p4 large file system.
1047        See https://git-lfs.github.com/ for details."""
1048
1049     def __init__(self, *args):
1050         LargeFileSystem.__init__(self, *args)
1051         self.baseGitAttributes = []
1052
1053     def generatePointer(self, contentFile):
1054         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055            mode and content which is stored in the Git repository instead of
1056            the actual content. Return also the new location of the actual
1057            content.
1058            """
1059         pointerProcess = subprocess.Popen(
1060             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061             stdout=subprocess.PIPE
1062         )
1063         pointerFile = pointerProcess.stdout.read()
1064         if pointerProcess.wait():
1065             os.remove(contentFile)
1066             die('git-lfs pointer command failed. Did you install the extension?')
1067         pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
1068         oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
1069         localLargeFile = os.path.join(
1070             os.getcwd(),
1071             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1072             oid,
1073         )
1074         # LFS Spec states that pointer files should not have the executable bit set.
1075         gitMode = '100644'
1076         return (gitMode, pointerContents, localLargeFile)
1077
1078     def pushFile(self, localLargeFile):
1079         uploadProcess = subprocess.Popen(
1080             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1081         )
1082         if uploadProcess.wait():
1083             die('git-lfs push command failed. Did you define a remote?')
1084
1085     def generateGitAttributes(self):
1086         return (
1087             self.baseGitAttributes +
1088             [
1089                 '\n',
1090                 '#\n',
1091                 '# Git LFS (see https://git-lfs.github.com/)\n',
1092                 '#\n',
1093             ] +
1094             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1095                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1096             ] +
1097             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1098                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1099             ]
1100         )
1101
1102     def addLargeFile(self, relPath):
1103         LargeFileSystem.addLargeFile(self, relPath)
1104         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1105
1106     def removeLargeFile(self, relPath):
1107         LargeFileSystem.removeLargeFile(self, relPath)
1108         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1109
1110     def processContent(self, git_mode, relPath, contents):
1111         if relPath == '.gitattributes':
1112             self.baseGitAttributes = contents
1113             return (git_mode, self.generateGitAttributes())
1114         else:
1115             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1116
1117 class Command:
1118     def __init__(self):
1119         self.usage = "usage: %prog [options]"
1120         self.needsGit = True
1121         self.verbose = False
1122
1123 class P4UserMap:
1124     def __init__(self):
1125         self.userMapFromPerforceServer = False
1126         self.myP4UserId = None
1127
1128     def p4UserId(self):
1129         if self.myP4UserId:
1130             return self.myP4UserId
1131
1132         results = p4CmdList("user -o")
1133         for r in results:
1134             if r.has_key('User'):
1135                 self.myP4UserId = r['User']
1136                 return r['User']
1137         die("Could not find your p4 user id")
1138
1139     def p4UserIsMe(self, p4User):
1140         # return True if the given p4 user is actually me
1141         me = self.p4UserId()
1142         if not p4User or p4User != me:
1143             return False
1144         else:
1145             return True
1146
1147     def getUserCacheFilename(self):
1148         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1149         return home + "/.gitp4-usercache.txt"
1150
1151     def getUserMapFromPerforceServer(self):
1152         if self.userMapFromPerforceServer:
1153             return
1154         self.users = {}
1155         self.emails = {}
1156
1157         for output in p4CmdList("users"):
1158             if not output.has_key("User"):
1159                 continue
1160             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1161             self.emails[output["Email"]] = output["User"]
1162
1163         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1164         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1165             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1166             if mapUser and len(mapUser[0]) == 3:
1167                 user = mapUser[0][0]
1168                 fullname = mapUser[0][1]
1169                 email = mapUser[0][2]
1170                 self.users[user] = fullname + " <" + email + ">"
1171                 self.emails[email] = user
1172
1173         s = ''
1174         for (key, val) in self.users.items():
1175             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1176
1177         open(self.getUserCacheFilename(), "wb").write(s)
1178         self.userMapFromPerforceServer = True
1179
1180     def loadUserMapFromCache(self):
1181         self.users = {}
1182         self.userMapFromPerforceServer = False
1183         try:
1184             cache = open(self.getUserCacheFilename(), "rb")
1185             lines = cache.readlines()
1186             cache.close()
1187             for line in lines:
1188                 entry = line.strip().split("\t")
1189                 self.users[entry[0]] = entry[1]
1190         except IOError:
1191             self.getUserMapFromPerforceServer()
1192
1193 class P4Debug(Command):
1194     def __init__(self):
1195         Command.__init__(self)
1196         self.options = []
1197         self.description = "A tool to debug the output of p4 -G."
1198         self.needsGit = False
1199
1200     def run(self, args):
1201         j = 0
1202         for output in p4CmdList(args):
1203             print 'Element: %d' % j
1204             j += 1
1205             print output
1206         return True
1207
1208 class P4RollBack(Command):
1209     def __init__(self):
1210         Command.__init__(self)
1211         self.options = [
1212             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1213         ]
1214         self.description = "A tool to debug the multi-branch import. Don't use :)"
1215         self.rollbackLocalBranches = False
1216
1217     def run(self, args):
1218         if len(args) != 1:
1219             return False
1220         maxChange = int(args[0])
1221
1222         if "p4ExitCode" in p4Cmd("changes -m 1"):
1223             die("Problems executing p4");
1224
1225         if self.rollbackLocalBranches:
1226             refPrefix = "refs/heads/"
1227             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1228         else:
1229             refPrefix = "refs/remotes/"
1230             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1231
1232         for line in lines:
1233             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1234                 line = line.strip()
1235                 ref = refPrefix + line
1236                 log = extractLogMessageFromGitCommit(ref)
1237                 settings = extractSettingsGitLog(log)
1238
1239                 depotPaths = settings['depot-paths']
1240                 change = settings['change']
1241
1242                 changed = False
1243
1244                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1245                                                            for p in depotPaths]))) == 0:
1246                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1247                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1248                     continue
1249
1250                 while change and int(change) > maxChange:
1251                     changed = True
1252                     if self.verbose:
1253                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1254                     system("git update-ref %s \"%s^\"" % (ref, ref))
1255                     log = extractLogMessageFromGitCommit(ref)
1256                     settings =  extractSettingsGitLog(log)
1257
1258
1259                     depotPaths = settings['depot-paths']
1260                     change = settings['change']
1261
1262                 if changed:
1263                     print "%s rewound to %s" % (ref, change)
1264
1265         return True
1266
1267 class P4Submit(Command, P4UserMap):
1268
1269     conflict_behavior_choices = ("ask", "skip", "quit")
1270
1271     def __init__(self):
1272         Command.__init__(self)
1273         P4UserMap.__init__(self)
1274         self.options = [
1275                 optparse.make_option("--origin", dest="origin"),
1276                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1277                 # preserve the user, requires relevant p4 permissions
1278                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1279                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1280                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1281                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1282                 optparse.make_option("--conflict", dest="conflict_behavior",
1283                                      choices=self.conflict_behavior_choices),
1284                 optparse.make_option("--branch", dest="branch"),
1285         ]
1286         self.description = "Submit changes from git to the perforce depot."
1287         self.usage += " [name of git branch to submit into perforce depot]"
1288         self.origin = ""
1289         self.detectRenames = False
1290         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1291         self.dry_run = False
1292         self.prepare_p4_only = False
1293         self.conflict_behavior = None
1294         self.isWindows = (platform.system() == "Windows")
1295         self.exportLabels = False
1296         self.p4HasMoveCommand = p4_has_move_command()
1297         self.branch = None
1298
1299         if gitConfig('git-p4.largeFileSystem'):
1300             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1301
1302     def check(self):
1303         if len(p4CmdList("opened ...")) > 0:
1304             die("You have files opened with perforce! Close them before starting the sync.")
1305
1306     def separate_jobs_from_description(self, message):
1307         """Extract and return a possible Jobs field in the commit
1308            message.  It goes into a separate section in the p4 change
1309            specification.
1310
1311            A jobs line starts with "Jobs:" and looks like a new field
1312            in a form.  Values are white-space separated on the same
1313            line or on following lines that start with a tab.
1314
1315            This does not parse and extract the full git commit message
1316            like a p4 form.  It just sees the Jobs: line as a marker
1317            to pass everything from then on directly into the p4 form,
1318            but outside the description section.
1319
1320            Return a tuple (stripped log message, jobs string)."""
1321
1322         m = re.search(r'^Jobs:', message, re.MULTILINE)
1323         if m is None:
1324             return (message, None)
1325
1326         jobtext = message[m.start():]
1327         stripped_message = message[:m.start()].rstrip()
1328         return (stripped_message, jobtext)
1329
1330     def prepareLogMessage(self, template, message, jobs):
1331         """Edits the template returned from "p4 change -o" to insert
1332            the message in the Description field, and the jobs text in
1333            the Jobs field."""
1334         result = ""
1335
1336         inDescriptionSection = False
1337
1338         for line in template.split("\n"):
1339             if line.startswith("#"):
1340                 result += line + "\n"
1341                 continue
1342
1343             if inDescriptionSection:
1344                 if line.startswith("Files:") or line.startswith("Jobs:"):
1345                     inDescriptionSection = False
1346                     # insert Jobs section
1347                     if jobs:
1348                         result += jobs + "\n"
1349                 else:
1350                     continue
1351             else:
1352                 if line.startswith("Description:"):
1353                     inDescriptionSection = True
1354                     line += "\n"
1355                     for messageLine in message.split("\n"):
1356                         line += "\t" + messageLine + "\n"
1357
1358             result += line + "\n"
1359
1360         return result
1361
1362     def patchRCSKeywords(self, file, pattern):
1363         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1364         (handle, outFileName) = tempfile.mkstemp(dir='.')
1365         try:
1366             outFile = os.fdopen(handle, "w+")
1367             inFile = open(file, "r")
1368             regexp = re.compile(pattern, re.VERBOSE)
1369             for line in inFile.readlines():
1370                 line = regexp.sub(r'$\1$', line)
1371                 outFile.write(line)
1372             inFile.close()
1373             outFile.close()
1374             # Forcibly overwrite the original file
1375             os.unlink(file)
1376             shutil.move(outFileName, file)
1377         except:
1378             # cleanup our temporary file
1379             os.unlink(outFileName)
1380             print "Failed to strip RCS keywords in %s" % file
1381             raise
1382
1383         print "Patched up RCS keywords in %s" % file
1384
1385     def p4UserForCommit(self,id):
1386         # Return the tuple (perforce user,git email) for a given git commit id
1387         self.getUserMapFromPerforceServer()
1388         gitEmail = read_pipe(["git", "log", "--max-count=1",
1389                               "--format=%ae", id])
1390         gitEmail = gitEmail.strip()
1391         if not self.emails.has_key(gitEmail):
1392             return (None,gitEmail)
1393         else:
1394             return (self.emails[gitEmail],gitEmail)
1395
1396     def checkValidP4Users(self,commits):
1397         # check if any git authors cannot be mapped to p4 users
1398         for id in commits:
1399             (user,email) = self.p4UserForCommit(id)
1400             if not user:
1401                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1402                 if gitConfigBool("git-p4.allowMissingP4Users"):
1403                     print "%s" % msg
1404                 else:
1405                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1406
1407     def lastP4Changelist(self):
1408         # Get back the last changelist number submitted in this client spec. This
1409         # then gets used to patch up the username in the change. If the same
1410         # client spec is being used by multiple processes then this might go
1411         # wrong.
1412         results = p4CmdList("client -o")        # find the current client
1413         client = None
1414         for r in results:
1415             if r.has_key('Client'):
1416                 client = r['Client']
1417                 break
1418         if not client:
1419             die("could not get client spec")
1420         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1421         for r in results:
1422             if r.has_key('change'):
1423                 return r['change']
1424         die("Could not get changelist number for last submit - cannot patch up user details")
1425
1426     def modifyChangelistUser(self, changelist, newUser):
1427         # fixup the user field of a changelist after it has been submitted.
1428         changes = p4CmdList("change -o %s" % changelist)
1429         if len(changes) != 1:
1430             die("Bad output from p4 change modifying %s to user %s" %
1431                 (changelist, newUser))
1432
1433         c = changes[0]
1434         if c['User'] == newUser: return   # nothing to do
1435         c['User'] = newUser
1436         input = marshal.dumps(c)
1437
1438         result = p4CmdList("change -f -i", stdin=input)
1439         for r in result:
1440             if r.has_key('code'):
1441                 if r['code'] == 'error':
1442                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1443             if r.has_key('data'):
1444                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1445                 return
1446         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1447
1448     def canChangeChangelists(self):
1449         # check to see if we have p4 admin or super-user permissions, either of
1450         # which are required to modify changelists.
1451         results = p4CmdList(["protects", self.depotPath])
1452         for r in results:
1453             if r.has_key('perm'):
1454                 if r['perm'] == 'admin':
1455                     return 1
1456                 if r['perm'] == 'super':
1457                     return 1
1458         return 0
1459
1460     def prepareSubmitTemplate(self):
1461         """Run "p4 change -o" to grab a change specification template.
1462            This does not use "p4 -G", as it is nice to keep the submission
1463            template in original order, since a human might edit it.
1464
1465            Remove lines in the Files section that show changes to files
1466            outside the depot path we're committing into."""
1467
1468         [upstream, settings] = findUpstreamBranchPoint()
1469
1470         template = ""
1471         inFilesSection = False
1472         for line in p4_read_pipe_lines(['change', '-o']):
1473             if line.endswith("\r\n"):
1474                 line = line[:-2] + "\n"
1475             if inFilesSection:
1476                 if line.startswith("\t"):
1477                     # path starts and ends with a tab
1478                     path = line[1:]
1479                     lastTab = path.rfind("\t")
1480                     if lastTab != -1:
1481                         path = path[:lastTab]
1482                         if settings.has_key('depot-paths'):
1483                             if not [p for p in settings['depot-paths']
1484                                     if p4PathStartsWith(path, p)]:
1485                                 continue
1486                         else:
1487                             if not p4PathStartsWith(path, self.depotPath):
1488                                 continue
1489                 else:
1490                     inFilesSection = False
1491             else:
1492                 if line.startswith("Files:"):
1493                     inFilesSection = True
1494
1495             template += line
1496
1497         return template
1498
1499     def edit_template(self, template_file):
1500         """Invoke the editor to let the user change the submission
1501            message.  Return true if okay to continue with the submit."""
1502
1503         # if configured to skip the editing part, just submit
1504         if gitConfigBool("git-p4.skipSubmitEdit"):
1505             return True
1506
1507         # look at the modification time, to check later if the user saved
1508         # the file
1509         mtime = os.stat(template_file).st_mtime
1510
1511         # invoke the editor
1512         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1513             editor = os.environ.get("P4EDITOR")
1514         else:
1515             editor = read_pipe("git var GIT_EDITOR").strip()
1516         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1517
1518         # If the file was not saved, prompt to see if this patch should
1519         # be skipped.  But skip this verification step if configured so.
1520         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1521             return True
1522
1523         # modification time updated means user saved the file
1524         if os.stat(template_file).st_mtime > mtime:
1525             return True
1526
1527         while True:
1528             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1529             if response == 'y':
1530                 return True
1531             if response == 'n':
1532                 return False
1533
1534     def get_diff_description(self, editedFiles, filesToAdd):
1535         # diff
1536         if os.environ.has_key("P4DIFF"):
1537             del(os.environ["P4DIFF"])
1538         diff = ""
1539         for editedFile in editedFiles:
1540             diff += p4_read_pipe(['diff', '-du',
1541                                   wildcard_encode(editedFile)])
1542
1543         # new file diff
1544         newdiff = ""
1545         for newFile in filesToAdd:
1546             newdiff += "==== new file ====\n"
1547             newdiff += "--- /dev/null\n"
1548             newdiff += "+++ %s\n" % newFile
1549             f = open(newFile, "r")
1550             for line in f.readlines():
1551                 newdiff += "+" + line
1552             f.close()
1553
1554         return (diff + newdiff).replace('\r\n', '\n')
1555
1556     def applyCommit(self, id):
1557         """Apply one commit, return True if it succeeded."""
1558
1559         print "Applying", read_pipe(["git", "show", "-s",
1560                                      "--format=format:%h %s", id])
1561
1562         (p4User, gitEmail) = self.p4UserForCommit(id)
1563
1564         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1565         filesToAdd = set()
1566         filesToChangeType = set()
1567         filesToDelete = set()
1568         editedFiles = set()
1569         pureRenameCopy = set()
1570         filesToChangeExecBit = {}
1571
1572         for line in diff:
1573             diff = parseDiffTreeEntry(line)
1574             modifier = diff['status']
1575             path = diff['src']
1576             if modifier == "M":
1577                 p4_edit(path)
1578                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1579                     filesToChangeExecBit[path] = diff['dst_mode']
1580                 editedFiles.add(path)
1581             elif modifier == "A":
1582                 filesToAdd.add(path)
1583                 filesToChangeExecBit[path] = diff['dst_mode']
1584                 if path in filesToDelete:
1585                     filesToDelete.remove(path)
1586             elif modifier == "D":
1587                 filesToDelete.add(path)
1588                 if path in filesToAdd:
1589                     filesToAdd.remove(path)
1590             elif modifier == "C":
1591                 src, dest = diff['src'], diff['dst']
1592                 p4_integrate(src, dest)
1593                 pureRenameCopy.add(dest)
1594                 if diff['src_sha1'] != diff['dst_sha1']:
1595                     p4_edit(dest)
1596                     pureRenameCopy.discard(dest)
1597                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1598                     p4_edit(dest)
1599                     pureRenameCopy.discard(dest)
1600                     filesToChangeExecBit[dest] = diff['dst_mode']
1601                 if self.isWindows:
1602                     # turn off read-only attribute
1603                     os.chmod(dest, stat.S_IWRITE)
1604                 os.unlink(dest)
1605                 editedFiles.add(dest)
1606             elif modifier == "R":
1607                 src, dest = diff['src'], diff['dst']
1608                 if self.p4HasMoveCommand:
1609                     p4_edit(src)        # src must be open before move
1610                     p4_move(src, dest)  # opens for (move/delete, move/add)
1611                 else:
1612                     p4_integrate(src, dest)
1613                     if diff['src_sha1'] != diff['dst_sha1']:
1614                         p4_edit(dest)
1615                     else:
1616                         pureRenameCopy.add(dest)
1617                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1618                     if not self.p4HasMoveCommand:
1619                         p4_edit(dest)   # with move: already open, writable
1620                     filesToChangeExecBit[dest] = diff['dst_mode']
1621                 if not self.p4HasMoveCommand:
1622                     if self.isWindows:
1623                         os.chmod(dest, stat.S_IWRITE)
1624                     os.unlink(dest)
1625                     filesToDelete.add(src)
1626                 editedFiles.add(dest)
1627             elif modifier == "T":
1628                 filesToChangeType.add(path)
1629             else:
1630                 die("unknown modifier %s for %s" % (modifier, path))
1631
1632         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1633         patchcmd = diffcmd + " | git apply "
1634         tryPatchCmd = patchcmd + "--check -"
1635         applyPatchCmd = patchcmd + "--check --apply -"
1636         patch_succeeded = True
1637
1638         if os.system(tryPatchCmd) != 0:
1639             fixed_rcs_keywords = False
1640             patch_succeeded = False
1641             print "Unfortunately applying the change failed!"
1642
1643             # Patch failed, maybe it's just RCS keyword woes. Look through
1644             # the patch to see if that's possible.
1645             if gitConfigBool("git-p4.attemptRCSCleanup"):
1646                 file = None
1647                 pattern = None
1648                 kwfiles = {}
1649                 for file in editedFiles | filesToDelete:
1650                     # did this file's delta contain RCS keywords?
1651                     pattern = p4_keywords_regexp_for_file(file)
1652
1653                     if pattern:
1654                         # this file is a possibility...look for RCS keywords.
1655                         regexp = re.compile(pattern, re.VERBOSE)
1656                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1657                             if regexp.search(line):
1658                                 if verbose:
1659                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1660                                 kwfiles[file] = pattern
1661                                 break
1662
1663                 for file in kwfiles:
1664                     if verbose:
1665                         print "zapping %s with %s" % (line,pattern)
1666                     # File is being deleted, so not open in p4.  Must
1667                     # disable the read-only bit on windows.
1668                     if self.isWindows and file not in editedFiles:
1669                         os.chmod(file, stat.S_IWRITE)
1670                     self.patchRCSKeywords(file, kwfiles[file])
1671                     fixed_rcs_keywords = True
1672
1673             if fixed_rcs_keywords:
1674                 print "Retrying the patch with RCS keywords cleaned up"
1675                 if os.system(tryPatchCmd) == 0:
1676                     patch_succeeded = True
1677
1678         if not patch_succeeded:
1679             for f in editedFiles:
1680                 p4_revert(f)
1681             return False
1682
1683         #
1684         # Apply the patch for real, and do add/delete/+x handling.
1685         #
1686         system(applyPatchCmd)
1687
1688         for f in filesToChangeType:
1689             p4_edit(f, "-t", "auto")
1690         for f in filesToAdd:
1691             p4_add(f)
1692         for f in filesToDelete:
1693             p4_revert(f)
1694             p4_delete(f)
1695
1696         # Set/clear executable bits
1697         for f in filesToChangeExecBit.keys():
1698             mode = filesToChangeExecBit[f]
1699             setP4ExecBit(f, mode)
1700
1701         #
1702         # Build p4 change description, starting with the contents
1703         # of the git commit message.
1704         #
1705         logMessage = extractLogMessageFromGitCommit(id)
1706         logMessage = logMessage.strip()
1707         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1708
1709         template = self.prepareSubmitTemplate()
1710         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1711
1712         if self.preserveUser:
1713            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1714
1715         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1716             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1717             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1718             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1719
1720         separatorLine = "######## everything below this line is just the diff #######\n"
1721         if not self.prepare_p4_only:
1722             submitTemplate += separatorLine
1723             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1724
1725         (handle, fileName) = tempfile.mkstemp()
1726         tmpFile = os.fdopen(handle, "w+b")
1727         if self.isWindows:
1728             submitTemplate = submitTemplate.replace("\n", "\r\n")
1729         tmpFile.write(submitTemplate)
1730         tmpFile.close()
1731
1732         if self.prepare_p4_only:
1733             #
1734             # Leave the p4 tree prepared, and the submit template around
1735             # and let the user decide what to do next
1736             #
1737             print
1738             print "P4 workspace prepared for submission."
1739             print "To submit or revert, go to client workspace"
1740             print "  " + self.clientPath
1741             print
1742             print "To submit, use \"p4 submit\" to write a new description,"
1743             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1744                   " \"git p4\"." % fileName
1745             print "You can delete the file \"%s\" when finished." % fileName
1746
1747             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1748                 print "To preserve change ownership by user %s, you must\n" \
1749                       "do \"p4 change -f <change>\" after submitting and\n" \
1750                       "edit the User field."
1751             if pureRenameCopy:
1752                 print "After submitting, renamed files must be re-synced."
1753                 print "Invoke \"p4 sync -f\" on each of these files:"
1754                 for f in pureRenameCopy:
1755                     print "  " + f
1756
1757             print
1758             print "To revert the changes, use \"p4 revert ...\", and delete"
1759             print "the submit template file \"%s\"" % fileName
1760             if filesToAdd:
1761                 print "Since the commit adds new files, they must be deleted:"
1762                 for f in filesToAdd:
1763                     print "  " + f
1764             print
1765             return True
1766
1767         #
1768         # Let the user edit the change description, then submit it.
1769         #
1770         submitted = False
1771
1772         try:
1773             if self.edit_template(fileName):
1774                 # read the edited message and submit
1775                 tmpFile = open(fileName, "rb")
1776                 message = tmpFile.read()
1777                 tmpFile.close()
1778                 if self.isWindows:
1779                     message = message.replace("\r\n", "\n")
1780                 submitTemplate = message[:message.index(separatorLine)]
1781                 p4_write_pipe(['submit', '-i'], submitTemplate)
1782
1783                 if self.preserveUser:
1784                     if p4User:
1785                         # Get last changelist number. Cannot easily get it from
1786                         # the submit command output as the output is
1787                         # unmarshalled.
1788                         changelist = self.lastP4Changelist()
1789                         self.modifyChangelistUser(changelist, p4User)
1790
1791                 # The rename/copy happened by applying a patch that created a
1792                 # new file.  This leaves it writable, which confuses p4.
1793                 for f in pureRenameCopy:
1794                     p4_sync(f, "-f")
1795                 submitted = True
1796
1797         finally:
1798             # skip this patch
1799             if not submitted:
1800                 print "Submission cancelled, undoing p4 changes."
1801                 for f in editedFiles:
1802                     p4_revert(f)
1803                 for f in filesToAdd:
1804                     p4_revert(f)
1805                     os.remove(f)
1806                 for f in filesToDelete:
1807                     p4_revert(f)
1808
1809         os.remove(fileName)
1810         return submitted
1811
1812     # Export git tags as p4 labels. Create a p4 label and then tag
1813     # with that.
1814     def exportGitTags(self, gitTags):
1815         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1816         if len(validLabelRegexp) == 0:
1817             validLabelRegexp = defaultLabelRegexp
1818         m = re.compile(validLabelRegexp)
1819
1820         for name in gitTags:
1821
1822             if not m.match(name):
1823                 if verbose:
1824                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1825                 continue
1826
1827             # Get the p4 commit this corresponds to
1828             logMessage = extractLogMessageFromGitCommit(name)
1829             values = extractSettingsGitLog(logMessage)
1830
1831             if not values.has_key('change'):
1832                 # a tag pointing to something not sent to p4; ignore
1833                 if verbose:
1834                     print "git tag %s does not give a p4 commit" % name
1835                 continue
1836             else:
1837                 changelist = values['change']
1838
1839             # Get the tag details.
1840             inHeader = True
1841             isAnnotated = False
1842             body = []
1843             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1844                 l = l.strip()
1845                 if inHeader:
1846                     if re.match(r'tag\s+', l):
1847                         isAnnotated = True
1848                     elif re.match(r'\s*$', l):
1849                         inHeader = False
1850                         continue
1851                 else:
1852                     body.append(l)
1853
1854             if not isAnnotated:
1855                 body = ["lightweight tag imported by git p4\n"]
1856
1857             # Create the label - use the same view as the client spec we are using
1858             clientSpec = getClientSpec()
1859
1860             labelTemplate  = "Label: %s\n" % name
1861             labelTemplate += "Description:\n"
1862             for b in body:
1863                 labelTemplate += "\t" + b + "\n"
1864             labelTemplate += "View:\n"
1865             for depot_side in clientSpec.mappings:
1866                 labelTemplate += "\t%s\n" % depot_side
1867
1868             if self.dry_run:
1869                 print "Would create p4 label %s for tag" % name
1870             elif self.prepare_p4_only:
1871                 print "Not creating p4 label %s for tag due to option" \
1872                       " --prepare-p4-only" % name
1873             else:
1874                 p4_write_pipe(["label", "-i"], labelTemplate)
1875
1876                 # Use the label
1877                 p4_system(["tag", "-l", name] +
1878                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1879
1880                 if verbose:
1881                     print "created p4 label for tag %s" % name
1882
1883     def run(self, args):
1884         if len(args) == 0:
1885             self.master = currentGitBranch()
1886         elif len(args) == 1:
1887             self.master = args[0]
1888             if not branchExists(self.master):
1889                 die("Branch %s does not exist" % self.master)
1890         else:
1891             return False
1892
1893         if self.master:
1894             allowSubmit = gitConfig("git-p4.allowSubmit")
1895             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1896                 die("%s is not in git-p4.allowSubmit" % self.master)
1897
1898         [upstream, settings] = findUpstreamBranchPoint()
1899         self.depotPath = settings['depot-paths'][0]
1900         if len(self.origin) == 0:
1901             self.origin = upstream
1902
1903         if self.preserveUser:
1904             if not self.canChangeChangelists():
1905                 die("Cannot preserve user names without p4 super-user or admin permissions")
1906
1907         # if not set from the command line, try the config file
1908         if self.conflict_behavior is None:
1909             val = gitConfig("git-p4.conflict")
1910             if val:
1911                 if val not in self.conflict_behavior_choices:
1912                     die("Invalid value '%s' for config git-p4.conflict" % val)
1913             else:
1914                 val = "ask"
1915             self.conflict_behavior = val
1916
1917         if self.verbose:
1918             print "Origin branch is " + self.origin
1919
1920         if len(self.depotPath) == 0:
1921             print "Internal error: cannot locate perforce depot path from existing branches"
1922             sys.exit(128)
1923
1924         self.useClientSpec = False
1925         if gitConfigBool("git-p4.useclientspec"):
1926             self.useClientSpec = True
1927         if self.useClientSpec:
1928             self.clientSpecDirs = getClientSpec()
1929
1930         # Check for the existance of P4 branches
1931         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1932
1933         if self.useClientSpec and not branchesDetected:
1934             # all files are relative to the client spec
1935             self.clientPath = getClientRoot()
1936         else:
1937             self.clientPath = p4Where(self.depotPath)
1938
1939         if self.clientPath == "":
1940             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1941
1942         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1943         self.oldWorkingDirectory = os.getcwd()
1944
1945         # ensure the clientPath exists
1946         new_client_dir = False
1947         if not os.path.exists(self.clientPath):
1948             new_client_dir = True
1949             os.makedirs(self.clientPath)
1950
1951         chdir(self.clientPath, is_client_path=True)
1952         if self.dry_run:
1953             print "Would synchronize p4 checkout in %s" % self.clientPath
1954         else:
1955             print "Synchronizing p4 checkout..."
1956             if new_client_dir:
1957                 # old one was destroyed, and maybe nobody told p4
1958                 p4_sync("...", "-f")
1959             else:
1960                 p4_sync("...")
1961         self.check()
1962
1963         commits = []
1964         if self.master:
1965             commitish = self.master
1966         else:
1967             commitish = 'HEAD'
1968
1969         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1970             commits.append(line.strip())
1971         commits.reverse()
1972
1973         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1974             self.checkAuthorship = False
1975         else:
1976             self.checkAuthorship = True
1977
1978         if self.preserveUser:
1979             self.checkValidP4Users(commits)
1980
1981         #
1982         # Build up a set of options to be passed to diff when
1983         # submitting each commit to p4.
1984         #
1985         if self.detectRenames:
1986             # command-line -M arg
1987             self.diffOpts = "-M"
1988         else:
1989             # If not explicitly set check the config variable
1990             detectRenames = gitConfig("git-p4.detectRenames")
1991
1992             if detectRenames.lower() == "false" or detectRenames == "":
1993                 self.diffOpts = ""
1994             elif detectRenames.lower() == "true":
1995                 self.diffOpts = "-M"
1996             else:
1997                 self.diffOpts = "-M%s" % detectRenames
1998
1999         # no command-line arg for -C or --find-copies-harder, just
2000         # config variables
2001         detectCopies = gitConfig("git-p4.detectCopies")
2002         if detectCopies.lower() == "false" or detectCopies == "":
2003             pass
2004         elif detectCopies.lower() == "true":
2005             self.diffOpts += " -C"
2006         else:
2007             self.diffOpts += " -C%s" % detectCopies
2008
2009         if gitConfigBool("git-p4.detectCopiesHarder"):
2010             self.diffOpts += " --find-copies-harder"
2011
2012         #
2013         # Apply the commits, one at a time.  On failure, ask if should
2014         # continue to try the rest of the patches, or quit.
2015         #
2016         if self.dry_run:
2017             print "Would apply"
2018         applied = []
2019         last = len(commits) - 1
2020         for i, commit in enumerate(commits):
2021             if self.dry_run:
2022                 print " ", read_pipe(["git", "show", "-s",
2023                                       "--format=format:%h %s", commit])
2024                 ok = True
2025             else:
2026                 ok = self.applyCommit(commit)
2027             if ok:
2028                 applied.append(commit)
2029             else:
2030                 if self.prepare_p4_only and i < last:
2031                     print "Processing only the first commit due to option" \
2032                           " --prepare-p4-only"
2033                     break
2034                 if i < last:
2035                     quit = False
2036                     while True:
2037                         # prompt for what to do, or use the option/variable
2038                         if self.conflict_behavior == "ask":
2039                             print "What do you want to do?"
2040                             response = raw_input("[s]kip this commit but apply"
2041                                                  " the rest, or [q]uit? ")
2042                             if not response:
2043                                 continue
2044                         elif self.conflict_behavior == "skip":
2045                             response = "s"
2046                         elif self.conflict_behavior == "quit":
2047                             response = "q"
2048                         else:
2049                             die("Unknown conflict_behavior '%s'" %
2050                                 self.conflict_behavior)
2051
2052                         if response[0] == "s":
2053                             print "Skipping this commit, but applying the rest"
2054                             break
2055                         if response[0] == "q":
2056                             print "Quitting"
2057                             quit = True
2058                             break
2059                     if quit:
2060                         break
2061
2062         chdir(self.oldWorkingDirectory)
2063
2064         if self.dry_run:
2065             pass
2066         elif self.prepare_p4_only:
2067             pass
2068         elif len(commits) == len(applied):
2069             print "All commits applied!"
2070
2071             sync = P4Sync()
2072             if self.branch:
2073                 sync.branch = self.branch
2074             sync.run([])
2075
2076             rebase = P4Rebase()
2077             rebase.rebase()
2078
2079         else:
2080             if len(applied) == 0:
2081                 print "No commits applied."
2082             else:
2083                 print "Applied only the commits marked with '*':"
2084                 for c in commits:
2085                     if c in applied:
2086                         star = "*"
2087                     else:
2088                         star = " "
2089                     print star, read_pipe(["git", "show", "-s",
2090                                            "--format=format:%h %s",  c])
2091                 print "You will have to do 'git p4 sync' and rebase."
2092
2093         if gitConfigBool("git-p4.exportLabels"):
2094             self.exportLabels = True
2095
2096         if self.exportLabels:
2097             p4Labels = getP4Labels(self.depotPath)
2098             gitTags = getGitTags()
2099
2100             missingGitTags = gitTags - p4Labels
2101             self.exportGitTags(missingGitTags)
2102
2103         # exit with error unless everything applied perfectly
2104         if len(commits) != len(applied):
2105                 sys.exit(1)
2106
2107         return True
2108
2109 class View(object):
2110     """Represent a p4 view ("p4 help views"), and map files in a
2111        repo according to the view."""
2112
2113     def __init__(self, client_name):
2114         self.mappings = []
2115         self.client_prefix = "//%s/" % client_name
2116         # cache results of "p4 where" to lookup client file locations
2117         self.client_spec_path_cache = {}
2118
2119     def append(self, view_line):
2120         """Parse a view line, splitting it into depot and client
2121            sides.  Append to self.mappings, preserving order.  This
2122            is only needed for tag creation."""
2123
2124         # Split the view line into exactly two words.  P4 enforces
2125         # structure on these lines that simplifies this quite a bit.
2126         #
2127         # Either or both words may be double-quoted.
2128         # Single quotes do not matter.
2129         # Double-quote marks cannot occur inside the words.
2130         # A + or - prefix is also inside the quotes.
2131         # There are no quotes unless they contain a space.
2132         # The line is already white-space stripped.
2133         # The two words are separated by a single space.
2134         #
2135         if view_line[0] == '"':
2136             # First word is double quoted.  Find its end.
2137             close_quote_index = view_line.find('"', 1)
2138             if close_quote_index <= 0:
2139                 die("No first-word closing quote found: %s" % view_line)
2140             depot_side = view_line[1:close_quote_index]
2141             # skip closing quote and space
2142             rhs_index = close_quote_index + 1 + 1
2143         else:
2144             space_index = view_line.find(" ")
2145             if space_index <= 0:
2146                 die("No word-splitting space found: %s" % view_line)
2147             depot_side = view_line[0:space_index]
2148             rhs_index = space_index + 1
2149
2150         # prefix + means overlay on previous mapping
2151         if depot_side.startswith("+"):
2152             depot_side = depot_side[1:]
2153
2154         # prefix - means exclude this path, leave out of mappings
2155         exclude = False
2156         if depot_side.startswith("-"):
2157             exclude = True
2158             depot_side = depot_side[1:]
2159
2160         if not exclude:
2161             self.mappings.append(depot_side)
2162
2163     def convert_client_path(self, clientFile):
2164         # chop off //client/ part to make it relative
2165         if not clientFile.startswith(self.client_prefix):
2166             die("No prefix '%s' on clientFile '%s'" %
2167                 (self.client_prefix, clientFile))
2168         return clientFile[len(self.client_prefix):]
2169
2170     def update_client_spec_path_cache(self, files):
2171         """ Caching file paths by "p4 where" batch query """
2172
2173         # List depot file paths exclude that already cached
2174         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2175
2176         if len(fileArgs) == 0:
2177             return  # All files in cache
2178
2179         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2180         for res in where_result:
2181             if "code" in res and res["code"] == "error":
2182                 # assume error is "... file(s) not in client view"
2183                 continue
2184             if "clientFile" not in res:
2185                 die("No clientFile in 'p4 where' output")
2186             if "unmap" in res:
2187                 # it will list all of them, but only one not unmap-ped
2188                 continue
2189             if gitConfigBool("core.ignorecase"):
2190                 res['depotFile'] = res['depotFile'].lower()
2191             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2192
2193         # not found files or unmap files set to ""
2194         for depotFile in fileArgs:
2195             if gitConfigBool("core.ignorecase"):
2196                 depotFile = depotFile.lower()
2197             if depotFile not in self.client_spec_path_cache:
2198                 self.client_spec_path_cache[depotFile] = ""
2199
2200     def map_in_client(self, depot_path):
2201         """Return the relative location in the client where this
2202            depot file should live.  Returns "" if the file should
2203            not be mapped in the client."""
2204
2205         if gitConfigBool("core.ignorecase"):
2206             depot_path = depot_path.lower()
2207
2208         if depot_path in self.client_spec_path_cache:
2209             return self.client_spec_path_cache[depot_path]
2210
2211         die( "Error: %s is not found in client spec path" % depot_path )
2212         return ""
2213
2214 class P4Sync(Command, P4UserMap):
2215     delete_actions = ( "delete", "move/delete", "purge" )
2216
2217     def __init__(self):
2218         Command.__init__(self)
2219         P4UserMap.__init__(self)
2220         self.options = [
2221                 optparse.make_option("--branch", dest="branch"),
2222                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2223                 optparse.make_option("--changesfile", dest="changesFile"),
2224                 optparse.make_option("--silent", dest="silent", action="store_true"),
2225                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2226                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2227                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2228                                      help="Import into refs/heads/ , not refs/remotes"),
2229                 optparse.make_option("--max-changes", dest="maxChanges",
2230                                      help="Maximum number of changes to import"),
2231                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2232                                      help="Internal block size to use when iteratively calling p4 changes"),
2233                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2234                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2235                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2236                                      help="Only sync files that are included in the Perforce Client Spec"),
2237                 optparse.make_option("-/", dest="cloneExclude",
2238                                      action="append", type="string",
2239                                      help="exclude depot path"),
2240         ]
2241         self.description = """Imports from Perforce into a git repository.\n
2242     example:
2243     //depot/my/project/ -- to import the current head
2244     //depot/my/project/@all -- to import everything
2245     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2246
2247     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2248
2249         self.usage += " //depot/path[@revRange]"
2250         self.silent = False
2251         self.createdBranches = set()
2252         self.committedChanges = set()
2253         self.branch = ""
2254         self.detectBranches = False
2255         self.detectLabels = False
2256         self.importLabels = False
2257         self.changesFile = ""
2258         self.syncWithOrigin = True
2259         self.importIntoRemotes = True
2260         self.maxChanges = ""
2261         self.changes_block_size = None
2262         self.keepRepoPath = False
2263         self.depotPaths = None
2264         self.p4BranchesInGit = []
2265         self.cloneExclude = []
2266         self.useClientSpec = False
2267         self.useClientSpec_from_options = False
2268         self.clientSpecDirs = None
2269         self.tempBranches = []
2270         self.tempBranchLocation = "git-p4-tmp"
2271         self.largeFileSystem = None
2272
2273         if gitConfig('git-p4.largeFileSystem'):
2274             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2275             self.largeFileSystem = largeFileSystemConstructor(
2276                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2277             )
2278
2279         if gitConfig("git-p4.syncFromOrigin") == "false":
2280             self.syncWithOrigin = False
2281
2282     # This is required for the "append" cloneExclude action
2283     def ensure_value(self, attr, value):
2284         if not hasattr(self, attr) or getattr(self, attr) is None:
2285             setattr(self, attr, value)
2286         return getattr(self, attr)
2287
2288     # Force a checkpoint in fast-import and wait for it to finish
2289     def checkpoint(self):
2290         self.gitStream.write("checkpoint\n\n")
2291         self.gitStream.write("progress checkpoint\n\n")
2292         out = self.gitOutput.readline()
2293         if self.verbose:
2294             print "checkpoint finished: " + out
2295
2296     def extractFilesFromCommit(self, commit):
2297         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2298                              for path in self.cloneExclude]
2299         files = []
2300         fnum = 0
2301         while commit.has_key("depotFile%s" % fnum):
2302             path =  commit["depotFile%s" % fnum]
2303
2304             if [p for p in self.cloneExclude
2305                 if p4PathStartsWith(path, p)]:
2306                 found = False
2307             else:
2308                 found = [p for p in self.depotPaths
2309                          if p4PathStartsWith(path, p)]
2310             if not found:
2311                 fnum = fnum + 1
2312                 continue
2313
2314             file = {}
2315             file["path"] = path
2316             file["rev"] = commit["rev%s" % fnum]
2317             file["action"] = commit["action%s" % fnum]
2318             file["type"] = commit["type%s" % fnum]
2319             files.append(file)
2320             fnum = fnum + 1
2321         return files
2322
2323     def extractJobsFromCommit(self, commit):
2324         jobs = []
2325         jnum = 0
2326         while commit.has_key("job%s" % jnum):
2327             job = commit["job%s" % jnum]
2328             jobs.append(job)
2329             jnum = jnum + 1
2330         return jobs
2331
2332     def stripRepoPath(self, path, prefixes):
2333         """When streaming files, this is called to map a p4 depot path
2334            to where it should go in git.  The prefixes are either
2335            self.depotPaths, or self.branchPrefixes in the case of
2336            branch detection."""
2337
2338         if self.useClientSpec:
2339             # branch detection moves files up a level (the branch name)
2340             # from what client spec interpretation gives
2341             path = self.clientSpecDirs.map_in_client(path)
2342             if self.detectBranches:
2343                 for b in self.knownBranches:
2344                     if path.startswith(b + "/"):
2345                         path = path[len(b)+1:]
2346
2347         elif self.keepRepoPath:
2348             # Preserve everything in relative path name except leading
2349             # //depot/; just look at first prefix as they all should
2350             # be in the same depot.
2351             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2352             if p4PathStartsWith(path, depot):
2353                 path = path[len(depot):]
2354
2355         else:
2356             for p in prefixes:
2357                 if p4PathStartsWith(path, p):
2358                     path = path[len(p):]
2359                     break
2360
2361         path = wildcard_decode(path)
2362         return path
2363
2364     def splitFilesIntoBranches(self, commit):
2365         """Look at each depotFile in the commit to figure out to what
2366            branch it belongs."""
2367
2368         if self.clientSpecDirs:
2369             files = self.extractFilesFromCommit(commit)
2370             self.clientSpecDirs.update_client_spec_path_cache(files)
2371
2372         branches = {}
2373         fnum = 0
2374         while commit.has_key("depotFile%s" % fnum):
2375             path =  commit["depotFile%s" % fnum]
2376             found = [p for p in self.depotPaths
2377                      if p4PathStartsWith(path, p)]
2378             if not found:
2379                 fnum = fnum + 1
2380                 continue
2381
2382             file = {}
2383             file["path"] = path
2384             file["rev"] = commit["rev%s" % fnum]
2385             file["action"] = commit["action%s" % fnum]
2386             file["type"] = commit["type%s" % fnum]
2387             fnum = fnum + 1
2388
2389             # start with the full relative path where this file would
2390             # go in a p4 client
2391             if self.useClientSpec:
2392                 relPath = self.clientSpecDirs.map_in_client(path)
2393             else:
2394                 relPath = self.stripRepoPath(path, self.depotPaths)
2395
2396             for branch in self.knownBranches.keys():
2397                 # add a trailing slash so that a commit into qt/4.2foo
2398                 # doesn't end up in qt/4.2, e.g.
2399                 if relPath.startswith(branch + "/"):
2400                     if branch not in branches:
2401                         branches[branch] = []
2402                     branches[branch].append(file)
2403                     break
2404
2405         return branches
2406
2407     def writeToGitStream(self, gitMode, relPath, contents):
2408         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2409         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2410         for d in contents:
2411             self.gitStream.write(d)
2412         self.gitStream.write('\n')
2413
2414     # output one file from the P4 stream
2415     # - helper for streamP4Files
2416
2417     def streamOneP4File(self, file, contents):
2418         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2419         if verbose:
2420             size = int(self.stream_file['fileSize'])
2421             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2422             sys.stdout.flush()
2423
2424         (type_base, type_mods) = split_p4_type(file["type"])
2425
2426         git_mode = "100644"
2427         if "x" in type_mods:
2428             git_mode = "100755"
2429         if type_base == "symlink":
2430             git_mode = "120000"
2431             # p4 print on a symlink sometimes contains "target\n";
2432             # if it does, remove the newline
2433             data = ''.join(contents)
2434             if not data:
2435                 # Some version of p4 allowed creating a symlink that pointed
2436                 # to nothing.  This causes p4 errors when checking out such
2437                 # a change, and errors here too.  Work around it by ignoring
2438                 # the bad symlink; hopefully a future change fixes it.
2439                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2440                 return
2441             elif data[-1] == '\n':
2442                 contents = [data[:-1]]
2443             else:
2444                 contents = [data]
2445
2446         if type_base == "utf16":
2447             # p4 delivers different text in the python output to -G
2448             # than it does when using "print -o", or normal p4 client
2449             # operations.  utf16 is converted to ascii or utf8, perhaps.
2450             # But ascii text saved as -t utf16 is completely mangled.
2451             # Invoke print -o to get the real contents.
2452             #
2453             # On windows, the newlines will always be mangled by print, so put
2454             # them back too.  This is not needed to the cygwin windows version,
2455             # just the native "NT" type.
2456             #
2457             try:
2458                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2459             except Exception as e:
2460                 if 'Translation of file content failed' in str(e):
2461                     type_base = 'binary'
2462                 else:
2463                     raise e
2464             else:
2465                 if p4_version_string().find('/NT') >= 0:
2466                     text = text.replace('\r\n', '\n')
2467                 contents = [ text ]
2468
2469         if type_base == "apple":
2470             # Apple filetype files will be streamed as a concatenation of
2471             # its appledouble header and the contents.  This is useless
2472             # on both macs and non-macs.  If using "print -q -o xx", it
2473             # will create "xx" with the data, and "%xx" with the header.
2474             # This is also not very useful.
2475             #
2476             # Ideally, someday, this script can learn how to generate
2477             # appledouble files directly and import those to git, but
2478             # non-mac machines can never find a use for apple filetype.
2479             print "\nIgnoring apple filetype file %s" % file['depotFile']
2480             return
2481
2482         # Note that we do not try to de-mangle keywords on utf16 files,
2483         # even though in theory somebody may want that.
2484         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2485         if pattern:
2486             regexp = re.compile(pattern, re.VERBOSE)
2487             text = ''.join(contents)
2488             text = regexp.sub(r'$\1$', text)
2489             contents = [ text ]
2490
2491         try:
2492             relPath.decode('ascii')
2493         except:
2494             encoding = 'utf8'
2495             if gitConfig('git-p4.pathEncoding'):
2496                 encoding = gitConfig('git-p4.pathEncoding')
2497             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2498             if self.verbose:
2499                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2500
2501         if self.largeFileSystem:
2502             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2503
2504         self.writeToGitStream(git_mode, relPath, contents)
2505
2506     def streamOneP4Deletion(self, file):
2507         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2508         if verbose:
2509             sys.stdout.write("delete %s\n" % relPath)
2510             sys.stdout.flush()
2511         self.gitStream.write("D %s\n" % relPath)
2512
2513         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2514             self.largeFileSystem.removeLargeFile(relPath)
2515
2516     # handle another chunk of streaming data
2517     def streamP4FilesCb(self, marshalled):
2518
2519         # catch p4 errors and complain
2520         err = None
2521         if "code" in marshalled:
2522             if marshalled["code"] == "error":
2523                 if "data" in marshalled:
2524                     err = marshalled["data"].rstrip()
2525
2526         if not err and 'fileSize' in self.stream_file:
2527             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2528             if required_bytes > 0:
2529                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2530                     os.getcwd(), required_bytes/1024/1024
2531                 )
2532
2533         if err:
2534             f = None
2535             if self.stream_have_file_info:
2536                 if "depotFile" in self.stream_file:
2537                     f = self.stream_file["depotFile"]
2538             # force a failure in fast-import, else an empty
2539             # commit will be made
2540             self.gitStream.write("\n")
2541             self.gitStream.write("die-now\n")
2542             self.gitStream.close()
2543             # ignore errors, but make sure it exits first
2544             self.importProcess.wait()
2545             if f:
2546                 die("Error from p4 print for %s: %s" % (f, err))
2547             else:
2548                 die("Error from p4 print: %s" % err)
2549
2550         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2551             # start of a new file - output the old one first
2552             self.streamOneP4File(self.stream_file, self.stream_contents)
2553             self.stream_file = {}
2554             self.stream_contents = []
2555             self.stream_have_file_info = False
2556
2557         # pick up the new file information... for the
2558         # 'data' field we need to append to our array
2559         for k in marshalled.keys():
2560             if k == 'data':
2561                 if 'streamContentSize' not in self.stream_file:
2562                     self.stream_file['streamContentSize'] = 0
2563                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2564                 self.stream_contents.append(marshalled['data'])
2565             else:
2566                 self.stream_file[k] = marshalled[k]
2567
2568         if (verbose and
2569             'streamContentSize' in self.stream_file and
2570             'fileSize' in self.stream_file and
2571             'depotFile' in self.stream_file):
2572             size = int(self.stream_file["fileSize"])
2573             if size > 0:
2574                 progress = 100*self.stream_file['streamContentSize']/size
2575                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2576                 sys.stdout.flush()
2577
2578         self.stream_have_file_info = True
2579
2580     # Stream directly from "p4 files" into "git fast-import"
2581     def streamP4Files(self, files):
2582         filesForCommit = []
2583         filesToRead = []
2584         filesToDelete = []
2585
2586         for f in files:
2587             filesForCommit.append(f)
2588             if f['action'] in self.delete_actions:
2589                 filesToDelete.append(f)
2590             else:
2591                 filesToRead.append(f)
2592
2593         # deleted files...
2594         for f in filesToDelete:
2595             self.streamOneP4Deletion(f)
2596
2597         if len(filesToRead) > 0:
2598             self.stream_file = {}
2599             self.stream_contents = []
2600             self.stream_have_file_info = False
2601
2602             # curry self argument
2603             def streamP4FilesCbSelf(entry):
2604                 self.streamP4FilesCb(entry)
2605
2606             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2607
2608             p4CmdList(["-x", "-", "print"],
2609                       stdin=fileArgs,
2610                       cb=streamP4FilesCbSelf)
2611
2612             # do the last chunk
2613             if self.stream_file.has_key('depotFile'):
2614                 self.streamOneP4File(self.stream_file, self.stream_contents)
2615
2616     def make_email(self, userid):
2617         if userid in self.users:
2618             return self.users[userid]
2619         else:
2620             return "%s <a@b>" % userid
2621
2622     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2623         """ Stream a p4 tag.
2624         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2625         """
2626
2627         if verbose:
2628             print "writing tag %s for commit %s" % (labelName, commit)
2629         gitStream.write("tag %s\n" % labelName)
2630         gitStream.write("from %s\n" % commit)
2631
2632         if labelDetails.has_key('Owner'):
2633             owner = labelDetails["Owner"]
2634         else:
2635             owner = None
2636
2637         # Try to use the owner of the p4 label, or failing that,
2638         # the current p4 user id.
2639         if owner:
2640             email = self.make_email(owner)
2641         else:
2642             email = self.make_email(self.p4UserId())
2643         tagger = "%s %s %s" % (email, epoch, self.tz)
2644
2645         gitStream.write("tagger %s\n" % tagger)
2646
2647         print "labelDetails=",labelDetails
2648         if labelDetails.has_key('Description'):
2649             description = labelDetails['Description']
2650         else:
2651             description = 'Label from git p4'
2652
2653         gitStream.write("data %d\n" % len(description))
2654         gitStream.write(description)
2655         gitStream.write("\n")
2656
2657     def inClientSpec(self, path):
2658         if not self.clientSpecDirs:
2659             return True
2660         inClientSpec = self.clientSpecDirs.map_in_client(path)
2661         if not inClientSpec and self.verbose:
2662             print('Ignoring file outside of client spec: {0}'.format(path))
2663         return inClientSpec
2664
2665     def hasBranchPrefix(self, path):
2666         if not self.branchPrefixes:
2667             return True
2668         hasPrefix = [p for p in self.branchPrefixes
2669                         if p4PathStartsWith(path, p)]
2670         if hasPrefix and self.verbose:
2671             print('Ignoring file outside of prefix: {0}'.format(path))
2672         return hasPrefix
2673
2674     def commit(self, details, files, branch, parent = ""):
2675         epoch = details["time"]
2676         author = details["user"]
2677         jobs = self.extractJobsFromCommit(details)
2678
2679         if self.verbose:
2680             print('commit into {0}'.format(branch))
2681
2682         if self.clientSpecDirs:
2683             self.clientSpecDirs.update_client_spec_path_cache(files)
2684
2685         files = [f for f in files
2686             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2687
2688         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2689             print('Ignoring revision {0} as it would produce an empty commit.'
2690                 .format(details['change']))
2691             return
2692
2693         self.gitStream.write("commit %s\n" % branch)
2694         self.gitStream.write("mark :%s\n" % details["change"])
2695         self.committedChanges.add(int(details["change"]))
2696         committer = ""
2697         if author not in self.users:
2698             self.getUserMapFromPerforceServer()
2699         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2700
2701         self.gitStream.write("committer %s\n" % committer)
2702
2703         self.gitStream.write("data <<EOT\n")
2704         self.gitStream.write(details["desc"])
2705         if len(jobs) > 0:
2706             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2707         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2708                              (','.join(self.branchPrefixes), details["change"]))
2709         if len(details['options']) > 0:
2710             self.gitStream.write(": options = %s" % details['options'])
2711         self.gitStream.write("]\nEOT\n\n")
2712
2713         if len(parent) > 0:
2714             if self.verbose:
2715                 print "parent %s" % parent
2716             self.gitStream.write("from %s\n" % parent)
2717
2718         self.streamP4Files(files)
2719         self.gitStream.write("\n")
2720
2721         change = int(details["change"])
2722
2723         if self.labels.has_key(change):
2724             label = self.labels[change]
2725             labelDetails = label[0]
2726             labelRevisions = label[1]
2727             if self.verbose:
2728                 print "Change %s is labelled %s" % (change, labelDetails)
2729
2730             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2731                                                 for p in self.branchPrefixes])
2732
2733             if len(files) == len(labelRevisions):
2734
2735                 cleanedFiles = {}
2736                 for info in files:
2737                     if info["action"] in self.delete_actions:
2738                         continue
2739                     cleanedFiles[info["depotFile"]] = info["rev"]
2740
2741                 if cleanedFiles == labelRevisions:
2742                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2743
2744                 else:
2745                     if not self.silent:
2746                         print ("Tag %s does not match with change %s: files do not match."
2747                                % (labelDetails["label"], change))
2748
2749             else:
2750                 if not self.silent:
2751                     print ("Tag %s does not match with change %s: file count is different."
2752                            % (labelDetails["label"], change))
2753
2754     # Build a dictionary of changelists and labels, for "detect-labels" option.
2755     def getLabels(self):
2756         self.labels = {}
2757
2758         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2759         if len(l) > 0 and not self.silent:
2760             print "Finding files belonging to labels in %s" % `self.depotPaths`
2761
2762         for output in l:
2763             label = output["label"]
2764             revisions = {}
2765             newestChange = 0
2766             if self.verbose:
2767                 print "Querying files for label %s" % label
2768             for file in p4CmdList(["files"] +
2769                                       ["%s...@%s" % (p, label)
2770                                           for p in self.depotPaths]):
2771                 revisions[file["depotFile"]] = file["rev"]
2772                 change = int(file["change"])
2773                 if change > newestChange:
2774                     newestChange = change
2775
2776             self.labels[newestChange] = [output, revisions]
2777
2778         if self.verbose:
2779             print "Label changes: %s" % self.labels.keys()
2780
2781     # Import p4 labels as git tags. A direct mapping does not
2782     # exist, so assume that if all the files are at the same revision
2783     # then we can use that, or it's something more complicated we should
2784     # just ignore.
2785     def importP4Labels(self, stream, p4Labels):
2786         if verbose:
2787             print "import p4 labels: " + ' '.join(p4Labels)
2788
2789         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2790         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2791         if len(validLabelRegexp) == 0:
2792             validLabelRegexp = defaultLabelRegexp
2793         m = re.compile(validLabelRegexp)
2794
2795         for name in p4Labels:
2796             commitFound = False
2797
2798             if not m.match(name):
2799                 if verbose:
2800                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2801                 continue
2802
2803             if name in ignoredP4Labels:
2804                 continue
2805
2806             labelDetails = p4CmdList(['label', "-o", name])[0]
2807
2808             # get the most recent changelist for each file in this label
2809             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2810                                 for p in self.depotPaths])
2811
2812             if change.has_key('change'):
2813                 # find the corresponding git commit; take the oldest commit
2814                 changelist = int(change['change'])
2815                 if changelist in self.committedChanges:
2816                     gitCommit = ":%d" % changelist       # use a fast-import mark
2817                     commitFound = True
2818                 else:
2819                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2820                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2821                     if len(gitCommit) == 0:
2822                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2823                     else:
2824                         commitFound = True
2825                         gitCommit = gitCommit.strip()
2826
2827                 if commitFound:
2828                     # Convert from p4 time format
2829                     try:
2830                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2831                     except ValueError:
2832                         print "Could not convert label time %s" % labelDetails['Update']
2833                         tmwhen = 1
2834
2835                     when = int(time.mktime(tmwhen))
2836                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2837                     if verbose:
2838                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2839             else:
2840                 if verbose:
2841                     print "Label %s has no changelists - possibly deleted?" % name
2842
2843             if not commitFound:
2844                 # We can't import this label; don't try again as it will get very
2845                 # expensive repeatedly fetching all the files for labels that will
2846                 # never be imported. If the label is moved in the future, the
2847                 # ignore will need to be removed manually.
2848                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2849
2850     def guessProjectName(self):
2851         for p in self.depotPaths:
2852             if p.endswith("/"):
2853                 p = p[:-1]
2854             p = p[p.strip().rfind("/") + 1:]
2855             if not p.endswith("/"):
2856                p += "/"
2857             return p
2858
2859     def getBranchMapping(self):
2860         lostAndFoundBranches = set()
2861
2862         user = gitConfig("git-p4.branchUser")
2863         if len(user) > 0:
2864             command = "branches -u %s" % user
2865         else:
2866             command = "branches"
2867
2868         for info in p4CmdList(command):
2869             details = p4Cmd(["branch", "-o", info["branch"]])
2870             viewIdx = 0
2871             while details.has_key("View%s" % viewIdx):
2872                 paths = details["View%s" % viewIdx].split(" ")
2873                 viewIdx = viewIdx + 1
2874                 # require standard //depot/foo/... //depot/bar/... mapping
2875                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2876                     continue
2877                 source = paths[0]
2878                 destination = paths[1]
2879                 ## HACK
2880                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2881                     source = source[len(self.depotPaths[0]):-4]
2882                     destination = destination[len(self.depotPaths[0]):-4]
2883
2884                     if destination in self.knownBranches:
2885                         if not self.silent:
2886                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2887                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2888                         continue
2889
2890                     self.knownBranches[destination] = source
2891
2892                     lostAndFoundBranches.discard(destination)
2893
2894                     if source not in self.knownBranches:
2895                         lostAndFoundBranches.add(source)
2896
2897         # Perforce does not strictly require branches to be defined, so we also
2898         # check git config for a branch list.
2899         #
2900         # Example of branch definition in git config file:
2901         # [git-p4]
2902         #   branchList=main:branchA
2903         #   branchList=main:branchB
2904         #   branchList=branchA:branchC
2905         configBranches = gitConfigList("git-p4.branchList")
2906         for branch in configBranches:
2907             if branch:
2908                 (source, destination) = branch.split(":")
2909                 self.knownBranches[destination] = source
2910
2911                 lostAndFoundBranches.discard(destination)
2912
2913                 if source not in self.knownBranches:
2914                     lostAndFoundBranches.add(source)
2915
2916
2917         for branch in lostAndFoundBranches:
2918             self.knownBranches[branch] = branch
2919
2920     def getBranchMappingFromGitBranches(self):
2921         branches = p4BranchesInGit(self.importIntoRemotes)
2922         for branch in branches.keys():
2923             if branch == "master":
2924                 branch = "main"
2925             else:
2926                 branch = branch[len(self.projectName):]
2927             self.knownBranches[branch] = branch
2928
2929     def updateOptionDict(self, d):
2930         option_keys = {}
2931         if self.keepRepoPath:
2932             option_keys['keepRepoPath'] = 1
2933
2934         d["options"] = ' '.join(sorted(option_keys.keys()))
2935
2936     def readOptions(self, d):
2937         self.keepRepoPath = (d.has_key('options')
2938                              and ('keepRepoPath' in d['options']))
2939
2940     def gitRefForBranch(self, branch):
2941         if branch == "main":
2942             return self.refPrefix + "master"
2943
2944         if len(branch) <= 0:
2945             return branch
2946
2947         return self.refPrefix + self.projectName + branch
2948
2949     def gitCommitByP4Change(self, ref, change):
2950         if self.verbose:
2951             print "looking in ref " + ref + " for change %s using bisect..." % change
2952
2953         earliestCommit = ""
2954         latestCommit = parseRevision(ref)
2955
2956         while True:
2957             if self.verbose:
2958                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2959             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2960             if len(next) == 0:
2961                 if self.verbose:
2962                     print "argh"
2963                 return ""
2964             log = extractLogMessageFromGitCommit(next)
2965             settings = extractSettingsGitLog(log)
2966             currentChange = int(settings['change'])
2967             if self.verbose:
2968                 print "current change %s" % currentChange
2969
2970             if currentChange == change:
2971                 if self.verbose:
2972                     print "found %s" % next
2973                 return next
2974
2975             if currentChange < change:
2976                 earliestCommit = "^%s" % next
2977             else:
2978                 latestCommit = "%s" % next
2979
2980         return ""
2981
2982     def importNewBranch(self, branch, maxChange):
2983         # make fast-import flush all changes to disk and update the refs using the checkpoint
2984         # command so that we can try to find the branch parent in the git history
2985         self.gitStream.write("checkpoint\n\n");
2986         self.gitStream.flush();
2987         branchPrefix = self.depotPaths[0] + branch + "/"
2988         range = "@1,%s" % maxChange
2989         #print "prefix" + branchPrefix
2990         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2991         if len(changes) <= 0:
2992             return False
2993         firstChange = changes[0]
2994         #print "first change in branch: %s" % firstChange
2995         sourceBranch = self.knownBranches[branch]
2996         sourceDepotPath = self.depotPaths[0] + sourceBranch
2997         sourceRef = self.gitRefForBranch(sourceBranch)
2998         #print "source " + sourceBranch
2999
3000         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3001         #print "branch parent: %s" % branchParentChange
3002         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3003         if len(gitParent) > 0:
3004             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3005             #print "parent git commit: %s" % gitParent
3006
3007         self.importChanges(changes)
3008         return True
3009
3010     def searchParent(self, parent, branch, target):
3011         parentFound = False
3012         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3013                                      "--no-merges", parent]):
3014             blob = blob.strip()
3015             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3016                 parentFound = True
3017                 if self.verbose:
3018                     print "Found parent of %s in commit %s" % (branch, blob)
3019                 break
3020         if parentFound:
3021             return blob
3022         else:
3023             return None
3024
3025     def importChanges(self, changes):
3026         cnt = 1
3027         for change in changes:
3028             description = p4_describe(change)
3029             self.updateOptionDict(description)
3030
3031             if not self.silent:
3032                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3033                 sys.stdout.flush()
3034             cnt = cnt + 1
3035
3036             try:
3037                 if self.detectBranches:
3038                     branches = self.splitFilesIntoBranches(description)
3039                     for branch in branches.keys():
3040                         ## HACK  --hwn
3041                         branchPrefix = self.depotPaths[0] + branch + "/"
3042                         self.branchPrefixes = [ branchPrefix ]
3043
3044                         parent = ""
3045
3046                         filesForCommit = branches[branch]
3047
3048                         if self.verbose:
3049                             print "branch is %s" % branch
3050
3051                         self.updatedBranches.add(branch)
3052
3053                         if branch not in self.createdBranches:
3054                             self.createdBranches.add(branch)
3055                             parent = self.knownBranches[branch]
3056                             if parent == branch:
3057                                 parent = ""
3058                             else:
3059                                 fullBranch = self.projectName + branch
3060                                 if fullBranch not in self.p4BranchesInGit:
3061                                     if not self.silent:
3062                                         print("\n    Importing new branch %s" % fullBranch);
3063                                     if self.importNewBranch(branch, change - 1):
3064                                         parent = ""
3065                                         self.p4BranchesInGit.append(fullBranch)
3066                                     if not self.silent:
3067                                         print("\n    Resuming with change %s" % change);
3068
3069                                 if self.verbose:
3070                                     print "parent determined through known branches: %s" % parent
3071
3072                         branch = self.gitRefForBranch(branch)
3073                         parent = self.gitRefForBranch(parent)
3074
3075                         if self.verbose:
3076                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3077
3078                         if len(parent) == 0 and branch in self.initialParents:
3079                             parent = self.initialParents[branch]
3080                             del self.initialParents[branch]
3081
3082                         blob = None
3083                         if len(parent) > 0:
3084                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3085                             if self.verbose:
3086                                 print "Creating temporary branch: " + tempBranch
3087                             self.commit(description, filesForCommit, tempBranch)
3088                             self.tempBranches.append(tempBranch)
3089                             self.checkpoint()
3090                             blob = self.searchParent(parent, branch, tempBranch)
3091                         if blob:
3092                             self.commit(description, filesForCommit, branch, blob)
3093                         else:
3094                             if self.verbose:
3095                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3096                             self.commit(description, filesForCommit, branch, parent)
3097                 else:
3098                     files = self.extractFilesFromCommit(description)
3099                     self.commit(description, files, self.branch,
3100                                 self.initialParent)
3101                     # only needed once, to connect to the previous commit
3102                     self.initialParent = ""
3103             except IOError:
3104                 print self.gitError.read()
3105                 sys.exit(1)
3106
3107     def importHeadRevision(self, revision):
3108         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3109
3110         details = {}
3111         details["user"] = "git perforce import user"
3112         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3113                            % (' '.join(self.depotPaths), revision))
3114         details["change"] = revision
3115         newestRevision = 0
3116
3117         fileCnt = 0
3118         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3119
3120         for info in p4CmdList(["files"] + fileArgs):
3121
3122             if 'code' in info and info['code'] == 'error':
3123                 sys.stderr.write("p4 returned an error: %s\n"
3124                                  % info['data'])
3125                 if info['data'].find("must refer to client") >= 0:
3126                     sys.stderr.write("This particular p4 error is misleading.\n")
3127                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3128                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3129                 sys.exit(1)
3130             if 'p4ExitCode' in info:
3131                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3132                 sys.exit(1)
3133
3134
3135             change = int(info["change"])
3136             if change > newestRevision:
3137                 newestRevision = change
3138
3139             if info["action"] in self.delete_actions:
3140                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3141                 #fileCnt = fileCnt + 1
3142                 continue
3143
3144             for prop in ["depotFile", "rev", "action", "type" ]:
3145                 details["%s%s" % (prop, fileCnt)] = info[prop]
3146
3147             fileCnt = fileCnt + 1
3148
3149         details["change"] = newestRevision
3150
3151         # Use time from top-most change so that all git p4 clones of
3152         # the same p4 repo have the same commit SHA1s.
3153         res = p4_describe(newestRevision)
3154         details["time"] = res["time"]
3155
3156         self.updateOptionDict(details)
3157         try:
3158             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3159         except IOError:
3160             print "IO error with git fast-import. Is your git version recent enough?"
3161             print self.gitError.read()
3162
3163
3164     def run(self, args):
3165         self.depotPaths = []
3166         self.changeRange = ""
3167         self.previousDepotPaths = []
3168         self.hasOrigin = False
3169
3170         # map from branch depot path to parent branch
3171         self.knownBranches = {}
3172         self.initialParents = {}
3173
3174         if self.importIntoRemotes:
3175             self.refPrefix = "refs/remotes/p4/"
3176         else:
3177             self.refPrefix = "refs/heads/p4/"
3178
3179         if self.syncWithOrigin:
3180             self.hasOrigin = originP4BranchesExist()
3181             if self.hasOrigin:
3182                 if not self.silent:
3183                     print 'Syncing with origin first, using "git fetch origin"'
3184                 system("git fetch origin")
3185
3186         branch_arg_given = bool(self.branch)
3187         if len(self.branch) == 0:
3188             self.branch = self.refPrefix + "master"
3189             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3190                 system("git update-ref %s refs/heads/p4" % self.branch)
3191                 system("git branch -D p4")
3192
3193         # accept either the command-line option, or the configuration variable
3194         if self.useClientSpec:
3195             # will use this after clone to set the variable
3196             self.useClientSpec_from_options = True
3197         else:
3198             if gitConfigBool("git-p4.useclientspec"):
3199                 self.useClientSpec = True
3200         if self.useClientSpec:
3201             self.clientSpecDirs = getClientSpec()
3202
3203         # TODO: should always look at previous commits,
3204         # merge with previous imports, if possible.
3205         if args == []:
3206             if self.hasOrigin:
3207                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3208
3209             # branches holds mapping from branch name to sha1
3210             branches = p4BranchesInGit(self.importIntoRemotes)
3211
3212             # restrict to just this one, disabling detect-branches
3213             if branch_arg_given:
3214                 short = self.branch.split("/")[-1]
3215                 if short in branches:
3216                     self.p4BranchesInGit = [ short ]
3217             else:
3218                 self.p4BranchesInGit = branches.keys()
3219
3220             if len(self.p4BranchesInGit) > 1:
3221                 if not self.silent:
3222                     print "Importing from/into multiple branches"
3223                 self.detectBranches = True
3224                 for branch in branches.keys():
3225                     self.initialParents[self.refPrefix + branch] = \
3226                         branches[branch]
3227
3228             if self.verbose:
3229                 print "branches: %s" % self.p4BranchesInGit
3230
3231             p4Change = 0
3232             for branch in self.p4BranchesInGit:
3233                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3234
3235                 settings = extractSettingsGitLog(logMsg)
3236
3237                 self.readOptions(settings)
3238                 if (settings.has_key('depot-paths')
3239                     and settings.has_key ('change')):
3240                     change = int(settings['change']) + 1
3241                     p4Change = max(p4Change, change)
3242
3243                     depotPaths = sorted(settings['depot-paths'])
3244                     if self.previousDepotPaths == []:
3245                         self.previousDepotPaths = depotPaths
3246                     else:
3247                         paths = []
3248                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3249                             prev_list = prev.split("/")
3250                             cur_list = cur.split("/")
3251                             for i in range(0, min(len(cur_list), len(prev_list))):
3252                                 if cur_list[i] <> prev_list[i]:
3253                                     i = i - 1
3254                                     break
3255
3256                             paths.append ("/".join(cur_list[:i + 1]))
3257
3258                         self.previousDepotPaths = paths
3259
3260             if p4Change > 0:
3261                 self.depotPaths = sorted(self.previousDepotPaths)
3262                 self.changeRange = "@%s,#head" % p4Change
3263                 if not self.silent and not self.detectBranches:
3264                     print "Performing incremental import into %s git branch" % self.branch
3265
3266         # accept multiple ref name abbreviations:
3267         #    refs/foo/bar/branch -> use it exactly
3268         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3269         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3270         if not self.branch.startswith("refs/"):
3271             if self.importIntoRemotes:
3272                 prepend = "refs/remotes/"
3273             else:
3274                 prepend = "refs/heads/"
3275             if not self.branch.startswith("p4/"):
3276                 prepend += "p4/"
3277             self.branch = prepend + self.branch
3278
3279         if len(args) == 0 and self.depotPaths:
3280             if not self.silent:
3281                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3282         else:
3283             if self.depotPaths and self.depotPaths != args:
3284                 print ("previous import used depot path %s and now %s was specified. "
3285                        "This doesn't work!" % (' '.join (self.depotPaths),
3286                                                ' '.join (args)))
3287                 sys.exit(1)
3288
3289             self.depotPaths = sorted(args)
3290
3291         revision = ""
3292         self.users = {}
3293
3294         # Make sure no revision specifiers are used when --changesfile
3295         # is specified.
3296         bad_changesfile = False
3297         if len(self.changesFile) > 0:
3298             for p in self.depotPaths:
3299                 if p.find("@") >= 0 or p.find("#") >= 0:
3300                     bad_changesfile = True
3301                     break
3302         if bad_changesfile:
3303             die("Option --changesfile is incompatible with revision specifiers")
3304
3305         newPaths = []
3306         for p in self.depotPaths:
3307             if p.find("@") != -1:
3308                 atIdx = p.index("@")
3309                 self.changeRange = p[atIdx:]
3310                 if self.changeRange == "@all":
3311                     self.changeRange = ""
3312                 elif ',' not in self.changeRange:
3313                     revision = self.changeRange
3314                     self.changeRange = ""
3315                 p = p[:atIdx]
3316             elif p.find("#") != -1:
3317                 hashIdx = p.index("#")
3318                 revision = p[hashIdx:]
3319                 p = p[:hashIdx]
3320             elif self.previousDepotPaths == []:
3321                 # pay attention to changesfile, if given, else import
3322                 # the entire p4 tree at the head revision
3323                 if len(self.changesFile) == 0:
3324                     revision = "#head"
3325
3326             p = re.sub ("\.\.\.$", "", p)
3327             if not p.endswith("/"):
3328                 p += "/"
3329
3330             newPaths.append(p)
3331
3332         self.depotPaths = newPaths
3333
3334         # --detect-branches may change this for each branch
3335         self.branchPrefixes = self.depotPaths
3336
3337         self.loadUserMapFromCache()
3338         self.labels = {}
3339         if self.detectLabels:
3340             self.getLabels();
3341
3342         if self.detectBranches:
3343             ## FIXME - what's a P4 projectName ?
3344             self.projectName = self.guessProjectName()
3345
3346             if self.hasOrigin:
3347                 self.getBranchMappingFromGitBranches()
3348             else:
3349                 self.getBranchMapping()
3350             if self.verbose:
3351                 print "p4-git branches: %s" % self.p4BranchesInGit
3352                 print "initial parents: %s" % self.initialParents
3353             for b in self.p4BranchesInGit:
3354                 if b != "master":
3355
3356                     ## FIXME
3357                     b = b[len(self.projectName):]
3358                 self.createdBranches.add(b)
3359
3360         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3361
3362         self.importProcess = subprocess.Popen(["git", "fast-import"],
3363                                               stdin=subprocess.PIPE,
3364                                               stdout=subprocess.PIPE,
3365                                               stderr=subprocess.PIPE);
3366         self.gitOutput = self.importProcess.stdout
3367         self.gitStream = self.importProcess.stdin
3368         self.gitError = self.importProcess.stderr
3369
3370         if revision:
3371             self.importHeadRevision(revision)
3372         else:
3373             changes = []
3374
3375             if len(self.changesFile) > 0:
3376                 output = open(self.changesFile).readlines()
3377                 changeSet = set()
3378                 for line in output:
3379                     changeSet.add(int(line))
3380
3381                 for change in changeSet:
3382                     changes.append(change)
3383
3384                 changes.sort()
3385             else:
3386                 # catch "git p4 sync" with no new branches, in a repo that
3387                 # does not have any existing p4 branches
3388                 if len(args) == 0:
3389                     if not self.p4BranchesInGit:
3390                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3391
3392                     # The default branch is master, unless --branch is used to
3393                     # specify something else.  Make sure it exists, or complain
3394                     # nicely about how to use --branch.
3395                     if not self.detectBranches:
3396                         if not branch_exists(self.branch):
3397                             if branch_arg_given:
3398                                 die("Error: branch %s does not exist." % self.branch)
3399                             else:
3400                                 die("Error: no branch %s; perhaps specify one with --branch." %
3401                                     self.branch)
3402
3403                 if self.verbose:
3404                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3405                                                               self.changeRange)
3406                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3407
3408                 if len(self.maxChanges) > 0:
3409                     changes = changes[:min(int(self.maxChanges), len(changes))]
3410
3411             if len(changes) == 0:
3412                 if not self.silent:
3413                     print "No changes to import!"
3414             else:
3415                 if not self.silent and not self.detectBranches:
3416                     print "Import destination: %s" % self.branch
3417
3418                 self.updatedBranches = set()
3419
3420                 if not self.detectBranches:
3421                     if args:
3422                         # start a new branch
3423                         self.initialParent = ""
3424                     else:
3425                         # build on a previous revision
3426                         self.initialParent = parseRevision(self.branch)
3427
3428                 self.importChanges(changes)
3429
3430                 if not self.silent:
3431                     print ""
3432                     if len(self.updatedBranches) > 0:
3433                         sys.stdout.write("Updated branches: ")
3434                         for b in self.updatedBranches:
3435                             sys.stdout.write("%s " % b)
3436                         sys.stdout.write("\n")
3437
3438         if gitConfigBool("git-p4.importLabels"):
3439             self.importLabels = True
3440
3441         if self.importLabels:
3442             p4Labels = getP4Labels(self.depotPaths)
3443             gitTags = getGitTags()
3444
3445             missingP4Labels = p4Labels - gitTags
3446             self.importP4Labels(self.gitStream, missingP4Labels)
3447
3448         self.gitStream.close()
3449         if self.importProcess.wait() != 0:
3450             die("fast-import failed: %s" % self.gitError.read())
3451         self.gitOutput.close()
3452         self.gitError.close()
3453
3454         # Cleanup temporary branches created during import
3455         if self.tempBranches != []:
3456             for branch in self.tempBranches:
3457                 read_pipe("git update-ref -d %s" % branch)
3458             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3459
3460         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3461         # a convenient shortcut refname "p4".
3462         if self.importIntoRemotes:
3463             head_ref = self.refPrefix + "HEAD"
3464             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3465                 system(["git", "symbolic-ref", head_ref, self.branch])
3466
3467         return True
3468
3469 class P4Rebase(Command):
3470     def __init__(self):
3471         Command.__init__(self)
3472         self.options = [
3473                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3474         ]
3475         self.importLabels = False
3476         self.description = ("Fetches the latest revision from perforce and "
3477                             + "rebases the current work (branch) against it")
3478
3479     def run(self, args):
3480         sync = P4Sync()
3481         sync.importLabels = self.importLabels
3482         sync.run([])
3483
3484         return self.rebase()
3485
3486     def rebase(self):
3487         if os.system("git update-index --refresh") != 0:
3488             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3489         if len(read_pipe("git diff-index HEAD --")) > 0:
3490             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3491
3492         [upstream, settings] = findUpstreamBranchPoint()
3493         if len(upstream) == 0:
3494             die("Cannot find upstream branchpoint for rebase")
3495
3496         # the branchpoint may be p4/foo~3, so strip off the parent
3497         upstream = re.sub("~[0-9]+$", "", upstream)
3498
3499         print "Rebasing the current branch onto %s" % upstream
3500         oldHead = read_pipe("git rev-parse HEAD").strip()
3501         system("git rebase %s" % upstream)
3502         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3503         return True
3504
3505 class P4Clone(P4Sync):
3506     def __init__(self):
3507         P4Sync.__init__(self)
3508         self.description = "Creates a new git repository and imports from Perforce into it"
3509         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3510         self.options += [
3511             optparse.make_option("--destination", dest="cloneDestination",
3512                                  action='store', default=None,
3513                                  help="where to leave result of the clone"),
3514             optparse.make_option("--bare", dest="cloneBare",
3515                                  action="store_true", default=False),
3516         ]
3517         self.cloneDestination = None
3518         self.needsGit = False
3519         self.cloneBare = False
3520
3521     def defaultDestination(self, args):
3522         ## TODO: use common prefix of args?
3523         depotPath = args[0]
3524         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3525         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3526         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3527         depotDir = re.sub(r"/$", "", depotDir)
3528         return os.path.split(depotDir)[1]
3529
3530     def run(self, args):
3531         if len(args) < 1:
3532             return False
3533
3534         if self.keepRepoPath and not self.cloneDestination:
3535             sys.stderr.write("Must specify destination for --keep-path\n")
3536             sys.exit(1)
3537
3538         depotPaths = args
3539
3540         if not self.cloneDestination and len(depotPaths) > 1:
3541             self.cloneDestination = depotPaths[-1]
3542             depotPaths = depotPaths[:-1]
3543
3544         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3545         for p in depotPaths:
3546             if not p.startswith("//"):
3547                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3548                 return False
3549
3550         if not self.cloneDestination:
3551             self.cloneDestination = self.defaultDestination(args)
3552
3553         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3554
3555         if not os.path.exists(self.cloneDestination):
3556             os.makedirs(self.cloneDestination)
3557         chdir(self.cloneDestination)
3558
3559         init_cmd = [ "git", "init" ]
3560         if self.cloneBare:
3561             init_cmd.append("--bare")
3562         retcode = subprocess.call(init_cmd)
3563         if retcode:
3564             raise CalledProcessError(retcode, init_cmd)
3565
3566         if not P4Sync.run(self, depotPaths):
3567             return False
3568
3569         # create a master branch and check out a work tree
3570         if gitBranchExists(self.branch):
3571             system([ "git", "branch", "master", self.branch ])
3572             if not self.cloneBare:
3573                 system([ "git", "checkout", "-f" ])
3574         else:
3575             print 'Not checking out any branch, use ' \
3576                   '"git checkout -q -b master <branch>"'
3577
3578         # auto-set this variable if invoked with --use-client-spec
3579         if self.useClientSpec_from_options:
3580             system("git config --bool git-p4.useclientspec true")
3581
3582         return True
3583
3584 class P4Branches(Command):
3585     def __init__(self):
3586         Command.__init__(self)
3587         self.options = [ ]
3588         self.description = ("Shows the git branches that hold imports and their "
3589                             + "corresponding perforce depot paths")
3590         self.verbose = False
3591
3592     def run(self, args):
3593         if originP4BranchesExist():
3594             createOrUpdateBranchesFromOrigin()
3595
3596         cmdline = "git rev-parse --symbolic "
3597         cmdline += " --remotes"
3598
3599         for line in read_pipe_lines(cmdline):
3600             line = line.strip()
3601
3602             if not line.startswith('p4/') or line == "p4/HEAD":
3603                 continue
3604             branch = line
3605
3606             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3607             settings = extractSettingsGitLog(log)
3608
3609             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3610         return True
3611
3612 class HelpFormatter(optparse.IndentedHelpFormatter):
3613     def __init__(self):
3614         optparse.IndentedHelpFormatter.__init__(self)
3615
3616     def format_description(self, description):
3617         if description:
3618             return description + "\n"
3619         else:
3620             return ""
3621
3622 def printUsage(commands):
3623     print "usage: %s <command> [options]" % sys.argv[0]
3624     print ""
3625     print "valid commands: %s" % ", ".join(commands)
3626     print ""
3627     print "Try %s <command> --help for command specific help." % sys.argv[0]
3628     print ""
3629
3630 commands = {
3631     "debug" : P4Debug,
3632     "submit" : P4Submit,
3633     "commit" : P4Submit,
3634     "sync" : P4Sync,
3635     "rebase" : P4Rebase,
3636     "clone" : P4Clone,
3637     "rollback" : P4RollBack,
3638     "branches" : P4Branches
3639 }
3640
3641
3642 def main():
3643     if len(sys.argv[1:]) == 0:
3644         printUsage(commands.keys())
3645         sys.exit(2)
3646
3647     cmdName = sys.argv[1]
3648     try:
3649         klass = commands[cmdName]
3650         cmd = klass()
3651     except KeyError:
3652         print "unknown command %s" % cmdName
3653         print ""
3654         printUsage(commands.keys())
3655         sys.exit(2)
3656
3657     options = cmd.options
3658     cmd.gitdir = os.environ.get("GIT_DIR", None)
3659
3660     args = sys.argv[2:]
3661
3662     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3663     if cmd.needsGit:
3664         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3665
3666     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3667                                    options,
3668                                    description = cmd.description,
3669                                    formatter = HelpFormatter())
3670
3671     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3672     global verbose
3673     verbose = cmd.verbose
3674     if cmd.needsGit:
3675         if cmd.gitdir == None:
3676             cmd.gitdir = os.path.abspath(".git")
3677             if not isValidGitDir(cmd.gitdir):
3678                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3679                 if os.path.exists(cmd.gitdir):
3680                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3681                     if len(cdup) > 0:
3682                         chdir(cdup);
3683
3684         if not isValidGitDir(cmd.gitdir):
3685             if isValidGitDir(cmd.gitdir + "/.git"):
3686                 cmd.gitdir += "/.git"
3687             else:
3688                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3689
3690         os.environ["GIT_DIR"] = cmd.gitdir
3691
3692     if not cmd.run(args):
3693         parser.print_help()
3694         sys.exit(2)
3695
3696
3697 if __name__ == '__main__':
3698     main()