Merge branch 'tg/doc-sec-list'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31     from subprocess import CalledProcessError
32 except ImportError:
33     # from python2.7:subprocess.py
34     # Exception classes used by this module.
35     class CalledProcessError(Exception):
36         """This exception is raised when a process run by check_call() returns
37         a non-zero exit status.  The exit status will be stored in the
38         returncode attribute."""
39         def __init__(self, returncode, cmd):
40             self.returncode = returncode
41             self.cmd = cmd
42         def __str__(self):
43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54     """Build a suitable p4 command line.
55
56     This consolidates building and returning a p4 command line into one
57     location. It means that hooking into the environment, or other configuration
58     can be done more easily.
59     """
60     real_cmd = ["p4"]
61
62     user = gitConfig("git-p4.user")
63     if len(user) > 0:
64         real_cmd += ["-u",user]
65
66     password = gitConfig("git-p4.password")
67     if len(password) > 0:
68         real_cmd += ["-P", password]
69
70     port = gitConfig("git-p4.port")
71     if len(port) > 0:
72         real_cmd += ["-p", port]
73
74     host = gitConfig("git-p4.host")
75     if len(host) > 0:
76         real_cmd += ["-H", host]
77
78     client = gitConfig("git-p4.client")
79     if len(client) > 0:
80         real_cmd += ["-c", client]
81
82     retries = gitConfigInt("git-p4.retries")
83     if retries is None:
84         # Perform 3 retries by default
85         retries = 3
86     if retries > 0:
87         # Provide a way to not pass this option by setting git-p4.retries to 0
88         real_cmd += ["-r", str(retries)]
89
90     if isinstance(cmd,basestring):
91         real_cmd = ' '.join(real_cmd) + ' ' + cmd
92     else:
93         real_cmd += cmd
94     return real_cmd
95
96 def git_dir(path):
97     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98         This won't automatically add ".git" to a directory.
99     """
100     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101     if not d or len(d) == 0:
102         return None
103     else:
104         return d
105
106 def chdir(path, is_client_path=False):
107     """Do chdir to the given path, and set the PWD environment
108        variable for use by P4.  It does not look at getcwd() output.
109        Since we're not using the shell, it is necessary to set the
110        PWD environment variable explicitly.
111
112        Normally, expand the path to force it to be absolute.  This
113        addresses the use of relative path names inside P4 settings,
114        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
115        as given; it looks for .p4config using PWD.
116
117        If is_client_path, the path was handed to us directly by p4,
118        and may be a symbolic link.  Do not call os.getcwd() in this
119        case, because it will cause p4 to think that PWD is not inside
120        the client path.
121        """
122
123     os.chdir(path)
124     if not is_client_path:
125         path = os.getcwd()
126     os.environ['PWD'] = path
127
128 def calcDiskFree():
129     """Return free space in bytes on the disk of the given dirname."""
130     if platform.system() == 'Windows':
131         free_bytes = ctypes.c_ulonglong(0)
132         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133         return free_bytes.value
134     else:
135         st = os.statvfs(os.getcwd())
136         return st.f_bavail * st.f_frsize
137
138 def die(msg):
139     if verbose:
140         raise Exception(msg)
141     else:
142         sys.stderr.write(msg + "\n")
143         sys.exit(1)
144
145 def write_pipe(c, stdin):
146     if verbose:
147         sys.stderr.write('Writing pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151     pipe = p.stdin
152     val = pipe.write(stdin)
153     pipe.close()
154     if p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_write_pipe(c, stdin):
160     real_cmd = p4_build_cmd(c)
161     return write_pipe(real_cmd, stdin)
162
163 def read_pipe_full(c):
164     """ Read output from  command. Returns a tuple
165         of the return status, stdout text and stderr
166         text.
167     """
168     if verbose:
169         sys.stderr.write('Reading pipe: %s\n' % str(c))
170
171     expand = isinstance(c,basestring)
172     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173     (out, err) = p.communicate()
174     return (p.returncode, out, err)
175
176 def read_pipe(c, ignore_error=False):
177     """ Read output from  command. Returns the output text on
178         success. On failure, terminates execution, unless
179         ignore_error is True, when it returns an empty string.
180     """
181     (retcode, out, err) = read_pipe_full(c)
182     if retcode != 0:
183         if ignore_error:
184             out = ""
185         else:
186             die('Command failed: %s\nError: %s' % (str(c), err))
187     return out
188
189 def read_pipe_text(c):
190     """ Read output from a command with trailing whitespace stripped.
191         On error, returns None.
192     """
193     (retcode, out, err) = read_pipe_full(c)
194     if retcode != 0:
195         return None
196     else:
197         return out.rstrip()
198
199 def p4_read_pipe(c, ignore_error=False):
200     real_cmd = p4_build_cmd(c)
201     return read_pipe(real_cmd, ignore_error)
202
203 def read_pipe_lines(c):
204     if verbose:
205         sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207     expand = isinstance(c, basestring)
208     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209     pipe = p.stdout
210     val = pipe.readlines()
211     if pipe.close() or p.wait():
212         die('Command failed: %s' % str(c))
213
214     return val
215
216 def p4_read_pipe_lines(c):
217     """Specifically invoke p4 on the command supplied. """
218     real_cmd = p4_build_cmd(c)
219     return read_pipe_lines(real_cmd)
220
221 def p4_has_command(cmd):
222     """Ask p4 for help on this command.  If it returns an error, the
223        command does not exist in this version of p4."""
224     real_cmd = p4_build_cmd(["help", cmd])
225     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226                                    stderr=subprocess.PIPE)
227     p.communicate()
228     return p.returncode == 0
229
230 def p4_has_move_command():
231     """See if the move command exists, that it supports -k, and that
232        it has not been administratively disabled.  The arguments
233        must be correct, but the filenames do not have to exist.  Use
234        ones with wildcards so even if they exist, it will fail."""
235
236     if not p4_has_command("move"):
237         return False
238     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240     (out, err) = p.communicate()
241     # return code will be 1 in either case
242     if err.find("Invalid option") >= 0:
243         return False
244     if err.find("disabled") >= 0:
245         return False
246     # assume it failed because @... was invalid changelist
247     return True
248
249 def system(cmd, ignore_error=False):
250     expand = isinstance(cmd,basestring)
251     if verbose:
252         sys.stderr.write("executing %s\n" % str(cmd))
253     retcode = subprocess.call(cmd, shell=expand)
254     if retcode and not ignore_error:
255         raise CalledProcessError(retcode, cmd)
256
257     return retcode
258
259 def p4_system(cmd):
260     """Specifically invoke p4 as the system command. """
261     real_cmd = p4_build_cmd(cmd)
262     expand = isinstance(real_cmd, basestring)
263     retcode = subprocess.call(real_cmd, shell=expand)
264     if retcode:
265         raise CalledProcessError(retcode, real_cmd)
266
267 _p4_version_string = None
268 def p4_version_string():
269     """Read the version string, showing just the last line, which
270        hopefully is the interesting version bit.
271
272        $ p4 -V
273        Perforce - The Fast Software Configuration Management System.
274        Copyright 1995-2011 Perforce Software.  All rights reserved.
275        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276     """
277     global _p4_version_string
278     if not _p4_version_string:
279         a = p4_read_pipe_lines(["-V"])
280         _p4_version_string = a[-1].rstrip()
281     return _p4_version_string
282
283 def p4_integrate(src, dest):
284     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
285
286 def p4_sync(f, *options):
287     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
288
289 def p4_add(f):
290     # forcibly add file names with wildcards
291     if wildcard_present(f):
292         p4_system(["add", "-f", f])
293     else:
294         p4_system(["add", f])
295
296 def p4_delete(f):
297     p4_system(["delete", wildcard_encode(f)])
298
299 def p4_edit(f, *options):
300     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
301
302 def p4_revert(f):
303     p4_system(["revert", wildcard_encode(f)])
304
305 def p4_reopen(type, f):
306     p4_system(["reopen", "-t", type, wildcard_encode(f)])
307
308 def p4_reopen_in_change(changelist, files):
309     cmd = ["reopen", "-c", str(changelist)] + files
310     p4_system(cmd)
311
312 def p4_move(src, dest):
313     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
315 def p4_last_change():
316     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317     return int(results[0]['change'])
318
319 def p4_describe(change, shelved=False):
320     """Make sure it returns a valid result by checking for
321        the presence of field "time".  Return a dict of the
322        results."""
323
324     cmd = ["describe", "-s"]
325     if shelved:
326         cmd += ["-S"]
327     cmd += [str(change)]
328
329     ds = p4CmdList(cmd, skip_info=True)
330     if len(ds) != 1:
331         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
332
333     d = ds[0]
334
335     if "p4ExitCode" in d:
336         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
337                                                       str(d)))
338     if "code" in d:
339         if d["code"] == "error":
340             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
341
342     if "time" not in d:
343         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
344
345     return d
346
347 #
348 # Canonicalize the p4 type and return a tuple of the
349 # base type, plus any modifiers.  See "p4 help filetypes"
350 # for a list and explanation.
351 #
352 def split_p4_type(p4type):
353
354     p4_filetypes_historical = {
355         "ctempobj": "binary+Sw",
356         "ctext": "text+C",
357         "cxtext": "text+Cx",
358         "ktext": "text+k",
359         "kxtext": "text+kx",
360         "ltext": "text+F",
361         "tempobj": "binary+FSw",
362         "ubinary": "binary+F",
363         "uresource": "resource+F",
364         "uxbinary": "binary+Fx",
365         "xbinary": "binary+x",
366         "xltext": "text+Fx",
367         "xtempobj": "binary+Swx",
368         "xtext": "text+x",
369         "xunicode": "unicode+x",
370         "xutf16": "utf16+x",
371     }
372     if p4type in p4_filetypes_historical:
373         p4type = p4_filetypes_historical[p4type]
374     mods = ""
375     s = p4type.split("+")
376     base = s[0]
377     mods = ""
378     if len(s) > 1:
379         mods = s[1]
380     return (base, mods)
381
382 #
383 # return the raw p4 type of a file (text, text+ko, etc)
384 #
385 def p4_type(f):
386     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
387     return results[0]['headType']
388
389 #
390 # Given a type base and modifier, return a regexp matching
391 # the keywords that can be expanded in the file
392 #
393 def p4_keywords_regexp_for_type(base, type_mods):
394     if base in ("text", "unicode", "binary"):
395         kwords = None
396         if "ko" in type_mods:
397             kwords = 'Id|Header'
398         elif "k" in type_mods:
399             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
400         else:
401             return None
402         pattern = r"""
403             \$              # Starts with a dollar, followed by...
404             (%s)            # one of the keywords, followed by...
405             (:[^$\n]+)?     # possibly an old expansion, followed by...
406             \$              # another dollar
407             """ % kwords
408         return pattern
409     else:
410         return None
411
412 #
413 # Given a file, return a regexp matching the possible
414 # RCS keywords that will be expanded, or None for files
415 # with kw expansion turned off.
416 #
417 def p4_keywords_regexp_for_file(file):
418     if not os.path.exists(file):
419         return None
420     else:
421         (type_base, type_mods) = split_p4_type(p4_type(file))
422         return p4_keywords_regexp_for_type(type_base, type_mods)
423
424 def setP4ExecBit(file, mode):
425     # Reopens an already open file and changes the execute bit to match
426     # the execute bit setting in the passed in mode.
427
428     p4Type = "+x"
429
430     if not isModeExec(mode):
431         p4Type = getP4OpenedType(file)
432         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
433         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
434         if p4Type[-1] == "+":
435             p4Type = p4Type[0:-1]
436
437     p4_reopen(p4Type, file)
438
439 def getP4OpenedType(file):
440     # Returns the perforce file type for the given file.
441
442     result = p4_read_pipe(["opened", wildcard_encode(file)])
443     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
444     if match:
445         return match.group(1)
446     else:
447         die("Could not determine file type for %s (result: '%s')" % (file, result))
448
449 # Return the set of all p4 labels
450 def getP4Labels(depotPaths):
451     labels = set()
452     if isinstance(depotPaths,basestring):
453         depotPaths = [depotPaths]
454
455     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
456         label = l['label']
457         labels.add(label)
458
459     return labels
460
461 # Return the set of all git tags
462 def getGitTags():
463     gitTags = set()
464     for line in read_pipe_lines(["git", "tag"]):
465         tag = line.strip()
466         gitTags.add(tag)
467     return gitTags
468
469 def diffTreePattern():
470     # This is a simple generator for the diff tree regex pattern. This could be
471     # a class variable if this and parseDiffTreeEntry were a part of a class.
472     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
473     while True:
474         yield pattern
475
476 def parseDiffTreeEntry(entry):
477     """Parses a single diff tree entry into its component elements.
478
479     See git-diff-tree(1) manpage for details about the format of the diff
480     output. This method returns a dictionary with the following elements:
481
482     src_mode - The mode of the source file
483     dst_mode - The mode of the destination file
484     src_sha1 - The sha1 for the source file
485     dst_sha1 - The sha1 fr the destination file
486     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
487     status_score - The score for the status (applicable for 'C' and 'R'
488                    statuses). This is None if there is no score.
489     src - The path for the source file.
490     dst - The path for the destination file. This is only present for
491           copy or renames. If it is not present, this is None.
492
493     If the pattern is not matched, None is returned."""
494
495     match = diffTreePattern().next().match(entry)
496     if match:
497         return {
498             'src_mode': match.group(1),
499             'dst_mode': match.group(2),
500             'src_sha1': match.group(3),
501             'dst_sha1': match.group(4),
502             'status': match.group(5),
503             'status_score': match.group(6),
504             'src': match.group(7),
505             'dst': match.group(10)
506         }
507     return None
508
509 def isModeExec(mode):
510     # Returns True if the given git mode represents an executable file,
511     # otherwise False.
512     return mode[-3:] == "755"
513
514 def isModeExecChanged(src_mode, dst_mode):
515     return isModeExec(src_mode) != isModeExec(dst_mode)
516
517 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
518
519     if isinstance(cmd,basestring):
520         cmd = "-G " + cmd
521         expand = True
522     else:
523         cmd = ["-G"] + cmd
524         expand = False
525
526     cmd = p4_build_cmd(cmd)
527     if verbose:
528         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
529
530     # Use a temporary file to avoid deadlocks without
531     # subprocess.communicate(), which would put another copy
532     # of stdout into memory.
533     stdin_file = None
534     if stdin is not None:
535         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
536         if isinstance(stdin,basestring):
537             stdin_file.write(stdin)
538         else:
539             for i in stdin:
540                 stdin_file.write(i + '\n')
541         stdin_file.flush()
542         stdin_file.seek(0)
543
544     p4 = subprocess.Popen(cmd,
545                           shell=expand,
546                           stdin=stdin_file,
547                           stdout=subprocess.PIPE)
548
549     result = []
550     try:
551         while True:
552             entry = marshal.load(p4.stdout)
553             if skip_info:
554                 if 'code' in entry and entry['code'] == 'info':
555                     continue
556             if cb is not None:
557                 cb(entry)
558             else:
559                 result.append(entry)
560     except EOFError:
561         pass
562     exitCode = p4.wait()
563     if exitCode != 0:
564         entry = {}
565         entry["p4ExitCode"] = exitCode
566         result.append(entry)
567
568     return result
569
570 def p4Cmd(cmd):
571     list = p4CmdList(cmd)
572     result = {}
573     for entry in list:
574         result.update(entry)
575     return result;
576
577 def p4Where(depotPath):
578     if not depotPath.endswith("/"):
579         depotPath += "/"
580     depotPathLong = depotPath + "..."
581     outputList = p4CmdList(["where", depotPathLong])
582     output = None
583     for entry in outputList:
584         if "depotFile" in entry:
585             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
586             # The base path always ends with "/...".
587             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
588                 output = entry
589                 break
590         elif "data" in entry:
591             data = entry.get("data")
592             space = data.find(" ")
593             if data[:space] == depotPath:
594                 output = entry
595                 break
596     if output == None:
597         return ""
598     if output["code"] == "error":
599         return ""
600     clientPath = ""
601     if "path" in output:
602         clientPath = output.get("path")
603     elif "data" in output:
604         data = output.get("data")
605         lastSpace = data.rfind(" ")
606         clientPath = data[lastSpace + 1:]
607
608     if clientPath.endswith("..."):
609         clientPath = clientPath[:-3]
610     return clientPath
611
612 def currentGitBranch():
613     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
614
615 def isValidGitDir(path):
616     return git_dir(path) != None
617
618 def parseRevision(ref):
619     return read_pipe("git rev-parse %s" % ref).strip()
620
621 def branchExists(ref):
622     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
623                      ignore_error=True)
624     return len(rev) > 0
625
626 def extractLogMessageFromGitCommit(commit):
627     logMessage = ""
628
629     ## fixme: title is first line of commit, not 1st paragraph.
630     foundTitle = False
631     for log in read_pipe_lines("git cat-file commit %s" % commit):
632        if not foundTitle:
633            if len(log) == 1:
634                foundTitle = True
635            continue
636
637        logMessage += log
638     return logMessage
639
640 def extractSettingsGitLog(log):
641     values = {}
642     for line in log.split("\n"):
643         line = line.strip()
644         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
645         if not m:
646             continue
647
648         assignments = m.group(1).split (':')
649         for a in assignments:
650             vals = a.split ('=')
651             key = vals[0].strip()
652             val = ('='.join (vals[1:])).strip()
653             if val.endswith ('\"') and val.startswith('"'):
654                 val = val[1:-1]
655
656             values[key] = val
657
658     paths = values.get("depot-paths")
659     if not paths:
660         paths = values.get("depot-path")
661     if paths:
662         values['depot-paths'] = paths.split(',')
663     return values
664
665 def gitBranchExists(branch):
666     proc = subprocess.Popen(["git", "rev-parse", branch],
667                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
668     return proc.wait() == 0;
669
670 def gitUpdateRef(ref, newvalue):
671     subprocess.check_call(["git", "update-ref", ref, newvalue])
672
673 def gitDeleteRef(ref):
674     subprocess.check_call(["git", "update-ref", "-d", ref])
675
676 _gitConfig = {}
677
678 def gitConfig(key, typeSpecifier=None):
679     if not _gitConfig.has_key(key):
680         cmd = [ "git", "config" ]
681         if typeSpecifier:
682             cmd += [ typeSpecifier ]
683         cmd += [ key ]
684         s = read_pipe(cmd, ignore_error=True)
685         _gitConfig[key] = s.strip()
686     return _gitConfig[key]
687
688 def gitConfigBool(key):
689     """Return a bool, using git config --bool.  It is True only if the
690        variable is set to true, and False if set to false or not present
691        in the config."""
692
693     if not _gitConfig.has_key(key):
694         _gitConfig[key] = gitConfig(key, '--bool') == "true"
695     return _gitConfig[key]
696
697 def gitConfigInt(key):
698     if not _gitConfig.has_key(key):
699         cmd = [ "git", "config", "--int", key ]
700         s = read_pipe(cmd, ignore_error=True)
701         v = s.strip()
702         try:
703             _gitConfig[key] = int(gitConfig(key, '--int'))
704         except ValueError:
705             _gitConfig[key] = None
706     return _gitConfig[key]
707
708 def gitConfigList(key):
709     if not _gitConfig.has_key(key):
710         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
711         _gitConfig[key] = s.strip().splitlines()
712         if _gitConfig[key] == ['']:
713             _gitConfig[key] = []
714     return _gitConfig[key]
715
716 def p4BranchesInGit(branchesAreInRemotes=True):
717     """Find all the branches whose names start with "p4/", looking
718        in remotes or heads as specified by the argument.  Return
719        a dictionary of { branch: revision } for each one found.
720        The branch names are the short names, without any
721        "p4/" prefix."""
722
723     branches = {}
724
725     cmdline = "git rev-parse --symbolic "
726     if branchesAreInRemotes:
727         cmdline += "--remotes"
728     else:
729         cmdline += "--branches"
730
731     for line in read_pipe_lines(cmdline):
732         line = line.strip()
733
734         # only import to p4/
735         if not line.startswith('p4/'):
736             continue
737         # special symbolic ref to p4/master
738         if line == "p4/HEAD":
739             continue
740
741         # strip off p4/ prefix
742         branch = line[len("p4/"):]
743
744         branches[branch] = parseRevision(line)
745
746     return branches
747
748 def branch_exists(branch):
749     """Make sure that the given ref name really exists."""
750
751     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
752     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
753     out, _ = p.communicate()
754     if p.returncode:
755         return False
756     # expect exactly one line of output: the branch name
757     return out.rstrip() == branch
758
759 def findUpstreamBranchPoint(head = "HEAD"):
760     branches = p4BranchesInGit()
761     # map from depot-path to branch name
762     branchByDepotPath = {}
763     for branch in branches.keys():
764         tip = branches[branch]
765         log = extractLogMessageFromGitCommit(tip)
766         settings = extractSettingsGitLog(log)
767         if settings.has_key("depot-paths"):
768             paths = ",".join(settings["depot-paths"])
769             branchByDepotPath[paths] = "remotes/p4/" + branch
770
771     settings = None
772     parent = 0
773     while parent < 65535:
774         commit = head + "~%s" % parent
775         log = extractLogMessageFromGitCommit(commit)
776         settings = extractSettingsGitLog(log)
777         if settings.has_key("depot-paths"):
778             paths = ",".join(settings["depot-paths"])
779             if branchByDepotPath.has_key(paths):
780                 return [branchByDepotPath[paths], settings]
781
782         parent = parent + 1
783
784     return ["", settings]
785
786 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
787     if not silent:
788         print ("Creating/updating branch(es) in %s based on origin branch(es)"
789                % localRefPrefix)
790
791     originPrefix = "origin/p4/"
792
793     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
794         line = line.strip()
795         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
796             continue
797
798         headName = line[len(originPrefix):]
799         remoteHead = localRefPrefix + headName
800         originHead = line
801
802         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
803         if (not original.has_key('depot-paths')
804             or not original.has_key('change')):
805             continue
806
807         update = False
808         if not gitBranchExists(remoteHead):
809             if verbose:
810                 print "creating %s" % remoteHead
811             update = True
812         else:
813             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
814             if settings.has_key('change') > 0:
815                 if settings['depot-paths'] == original['depot-paths']:
816                     originP4Change = int(original['change'])
817                     p4Change = int(settings['change'])
818                     if originP4Change > p4Change:
819                         print ("%s (%s) is newer than %s (%s). "
820                                "Updating p4 branch from origin."
821                                % (originHead, originP4Change,
822                                   remoteHead, p4Change))
823                         update = True
824                 else:
825                     print ("Ignoring: %s was imported from %s while "
826                            "%s was imported from %s"
827                            % (originHead, ','.join(original['depot-paths']),
828                               remoteHead, ','.join(settings['depot-paths'])))
829
830         if update:
831             system("git update-ref %s %s" % (remoteHead, originHead))
832
833 def originP4BranchesExist():
834         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
835
836
837 def p4ParseNumericChangeRange(parts):
838     changeStart = int(parts[0][1:])
839     if parts[1] == '#head':
840         changeEnd = p4_last_change()
841     else:
842         changeEnd = int(parts[1])
843
844     return (changeStart, changeEnd)
845
846 def chooseBlockSize(blockSize):
847     if blockSize:
848         return blockSize
849     else:
850         return defaultBlockSize
851
852 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
853     assert depotPaths
854
855     # Parse the change range into start and end. Try to find integer
856     # revision ranges as these can be broken up into blocks to avoid
857     # hitting server-side limits (maxrows, maxscanresults). But if
858     # that doesn't work, fall back to using the raw revision specifier
859     # strings, without using block mode.
860
861     if changeRange is None or changeRange == '':
862         changeStart = 1
863         changeEnd = p4_last_change()
864         block_size = chooseBlockSize(requestedBlockSize)
865     else:
866         parts = changeRange.split(',')
867         assert len(parts) == 2
868         try:
869             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
870             block_size = chooseBlockSize(requestedBlockSize)
871         except:
872             changeStart = parts[0][1:]
873             changeEnd = parts[1]
874             if requestedBlockSize:
875                 die("cannot use --changes-block-size with non-numeric revisions")
876             block_size = None
877
878     changes = set()
879
880     # Retrieve changes a block at a time, to prevent running
881     # into a MaxResults/MaxScanRows error from the server.
882
883     while True:
884         cmd = ['changes']
885
886         if block_size:
887             end = min(changeEnd, changeStart + block_size)
888             revisionRange = "%d,%d" % (changeStart, end)
889         else:
890             revisionRange = "%s,%s" % (changeStart, changeEnd)
891
892         for p in depotPaths:
893             cmd += ["%s...@%s" % (p, revisionRange)]
894
895         # Insert changes in chronological order
896         for entry in reversed(p4CmdList(cmd)):
897             if entry.has_key('p4ExitCode'):
898                 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
899             if not entry.has_key('change'):
900                 continue
901             changes.add(int(entry['change']))
902
903         if not block_size:
904             break
905
906         if end >= changeEnd:
907             break
908
909         changeStart = end + 1
910
911     changes = sorted(changes)
912     return changes
913
914 def p4PathStartsWith(path, prefix):
915     # This method tries to remedy a potential mixed-case issue:
916     #
917     # If UserA adds  //depot/DirA/file1
918     # and UserB adds //depot/dira/file2
919     #
920     # we may or may not have a problem. If you have core.ignorecase=true,
921     # we treat DirA and dira as the same directory
922     if gitConfigBool("core.ignorecase"):
923         return path.lower().startswith(prefix.lower())
924     return path.startswith(prefix)
925
926 def getClientSpec():
927     """Look at the p4 client spec, create a View() object that contains
928        all the mappings, and return it."""
929
930     specList = p4CmdList("client -o")
931     if len(specList) != 1:
932         die('Output from "client -o" is %d lines, expecting 1' %
933             len(specList))
934
935     # dictionary of all client parameters
936     entry = specList[0]
937
938     # the //client/ name
939     client_name = entry["Client"]
940
941     # just the keys that start with "View"
942     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
943
944     # hold this new View
945     view = View(client_name)
946
947     # append the lines, in order, to the view
948     for view_num in range(len(view_keys)):
949         k = "View%d" % view_num
950         if k not in view_keys:
951             die("Expected view key %s missing" % k)
952         view.append(entry[k])
953
954     return view
955
956 def getClientRoot():
957     """Grab the client directory."""
958
959     output = p4CmdList("client -o")
960     if len(output) != 1:
961         die('Output from "client -o" is %d lines, expecting 1' % len(output))
962
963     entry = output[0]
964     if "Root" not in entry:
965         die('Client has no "Root"')
966
967     return entry["Root"]
968
969 #
970 # P4 wildcards are not allowed in filenames.  P4 complains
971 # if you simply add them, but you can force it with "-f", in
972 # which case it translates them into %xx encoding internally.
973 #
974 def wildcard_decode(path):
975     # Search for and fix just these four characters.  Do % last so
976     # that fixing it does not inadvertently create new %-escapes.
977     # Cannot have * in a filename in windows; untested as to
978     # what p4 would do in such a case.
979     if not platform.system() == "Windows":
980         path = path.replace("%2A", "*")
981     path = path.replace("%23", "#") \
982                .replace("%40", "@") \
983                .replace("%25", "%")
984     return path
985
986 def wildcard_encode(path):
987     # do % first to avoid double-encoding the %s introduced here
988     path = path.replace("%", "%25") \
989                .replace("*", "%2A") \
990                .replace("#", "%23") \
991                .replace("@", "%40")
992     return path
993
994 def wildcard_present(path):
995     m = re.search("[*#@%]", path)
996     return m is not None
997
998 class LargeFileSystem(object):
999     """Base class for large file system support."""
1000
1001     def __init__(self, writeToGitStream):
1002         self.largeFiles = set()
1003         self.writeToGitStream = writeToGitStream
1004
1005     def generatePointer(self, cloneDestination, contentFile):
1006         """Return the content of a pointer file that is stored in Git instead of
1007            the actual content."""
1008         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1009
1010     def pushFile(self, localLargeFile):
1011         """Push the actual content which is not stored in the Git repository to
1012            a server."""
1013         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1014
1015     def hasLargeFileExtension(self, relPath):
1016         return reduce(
1017             lambda a, b: a or b,
1018             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1019             False
1020         )
1021
1022     def generateTempFile(self, contents):
1023         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1024         for d in contents:
1025             contentFile.write(d)
1026         contentFile.close()
1027         return contentFile.name
1028
1029     def exceedsLargeFileThreshold(self, relPath, contents):
1030         if gitConfigInt('git-p4.largeFileThreshold'):
1031             contentsSize = sum(len(d) for d in contents)
1032             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1033                 return True
1034         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1035             contentsSize = sum(len(d) for d in contents)
1036             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1037                 return False
1038             contentTempFile = self.generateTempFile(contents)
1039             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1040             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1041             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1042             zf.close()
1043             compressedContentsSize = zf.infolist()[0].compress_size
1044             os.remove(contentTempFile)
1045             os.remove(compressedContentFile.name)
1046             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1047                 return True
1048         return False
1049
1050     def addLargeFile(self, relPath):
1051         self.largeFiles.add(relPath)
1052
1053     def removeLargeFile(self, relPath):
1054         self.largeFiles.remove(relPath)
1055
1056     def isLargeFile(self, relPath):
1057         return relPath in self.largeFiles
1058
1059     def processContent(self, git_mode, relPath, contents):
1060         """Processes the content of git fast import. This method decides if a
1061            file is stored in the large file system and handles all necessary
1062            steps."""
1063         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1064             contentTempFile = self.generateTempFile(contents)
1065             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1066             if pointer_git_mode:
1067                 git_mode = pointer_git_mode
1068             if localLargeFile:
1069                 # Move temp file to final location in large file system
1070                 largeFileDir = os.path.dirname(localLargeFile)
1071                 if not os.path.isdir(largeFileDir):
1072                     os.makedirs(largeFileDir)
1073                 shutil.move(contentTempFile, localLargeFile)
1074                 self.addLargeFile(relPath)
1075                 if gitConfigBool('git-p4.largeFilePush'):
1076                     self.pushFile(localLargeFile)
1077                 if verbose:
1078                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1079         return (git_mode, contents)
1080
1081 class MockLFS(LargeFileSystem):
1082     """Mock large file system for testing."""
1083
1084     def generatePointer(self, contentFile):
1085         """The pointer content is the original content prefixed with "pointer-".
1086            The local filename of the large file storage is derived from the file content.
1087            """
1088         with open(contentFile, 'r') as f:
1089             content = next(f)
1090             gitMode = '100644'
1091             pointerContents = 'pointer-' + content
1092             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1093             return (gitMode, pointerContents, localLargeFile)
1094
1095     def pushFile(self, localLargeFile):
1096         """The remote filename of the large file storage is the same as the local
1097            one but in a different directory.
1098            """
1099         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1100         if not os.path.exists(remotePath):
1101             os.makedirs(remotePath)
1102         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1103
1104 class GitLFS(LargeFileSystem):
1105     """Git LFS as backend for the git-p4 large file system.
1106        See https://git-lfs.github.com/ for details."""
1107
1108     def __init__(self, *args):
1109         LargeFileSystem.__init__(self, *args)
1110         self.baseGitAttributes = []
1111
1112     def generatePointer(self, contentFile):
1113         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1114            mode and content which is stored in the Git repository instead of
1115            the actual content. Return also the new location of the actual
1116            content.
1117            """
1118         if os.path.getsize(contentFile) == 0:
1119             return (None, '', None)
1120
1121         pointerProcess = subprocess.Popen(
1122             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1123             stdout=subprocess.PIPE
1124         )
1125         pointerFile = pointerProcess.stdout.read()
1126         if pointerProcess.wait():
1127             os.remove(contentFile)
1128             die('git-lfs pointer command failed. Did you install the extension?')
1129
1130         # Git LFS removed the preamble in the output of the 'pointer' command
1131         # starting from version 1.2.0. Check for the preamble here to support
1132         # earlier versions.
1133         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1134         if pointerFile.startswith('Git LFS pointer for'):
1135             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1136
1137         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1138         localLargeFile = os.path.join(
1139             os.getcwd(),
1140             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1141             oid,
1142         )
1143         # LFS Spec states that pointer files should not have the executable bit set.
1144         gitMode = '100644'
1145         return (gitMode, pointerFile, localLargeFile)
1146
1147     def pushFile(self, localLargeFile):
1148         uploadProcess = subprocess.Popen(
1149             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1150         )
1151         if uploadProcess.wait():
1152             die('git-lfs push command failed. Did you define a remote?')
1153
1154     def generateGitAttributes(self):
1155         return (
1156             self.baseGitAttributes +
1157             [
1158                 '\n',
1159                 '#\n',
1160                 '# Git LFS (see https://git-lfs.github.com/)\n',
1161                 '#\n',
1162             ] +
1163             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1164                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1165             ] +
1166             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1167                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1168             ]
1169         )
1170
1171     def addLargeFile(self, relPath):
1172         LargeFileSystem.addLargeFile(self, relPath)
1173         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1174
1175     def removeLargeFile(self, relPath):
1176         LargeFileSystem.removeLargeFile(self, relPath)
1177         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1178
1179     def processContent(self, git_mode, relPath, contents):
1180         if relPath == '.gitattributes':
1181             self.baseGitAttributes = contents
1182             return (git_mode, self.generateGitAttributes())
1183         else:
1184             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1185
1186 class Command:
1187     def __init__(self):
1188         self.usage = "usage: %prog [options]"
1189         self.needsGit = True
1190         self.verbose = False
1191
1192     # This is required for the "append" cloneExclude action
1193     def ensure_value(self, attr, value):
1194         if not hasattr(self, attr) or getattr(self, attr) is None:
1195             setattr(self, attr, value)
1196         return getattr(self, attr)
1197
1198 class P4UserMap:
1199     def __init__(self):
1200         self.userMapFromPerforceServer = False
1201         self.myP4UserId = None
1202
1203     def p4UserId(self):
1204         if self.myP4UserId:
1205             return self.myP4UserId
1206
1207         results = p4CmdList("user -o")
1208         for r in results:
1209             if r.has_key('User'):
1210                 self.myP4UserId = r['User']
1211                 return r['User']
1212         die("Could not find your p4 user id")
1213
1214     def p4UserIsMe(self, p4User):
1215         # return True if the given p4 user is actually me
1216         me = self.p4UserId()
1217         if not p4User or p4User != me:
1218             return False
1219         else:
1220             return True
1221
1222     def getUserCacheFilename(self):
1223         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1224         return home + "/.gitp4-usercache.txt"
1225
1226     def getUserMapFromPerforceServer(self):
1227         if self.userMapFromPerforceServer:
1228             return
1229         self.users = {}
1230         self.emails = {}
1231
1232         for output in p4CmdList("users"):
1233             if not output.has_key("User"):
1234                 continue
1235             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1236             self.emails[output["Email"]] = output["User"]
1237
1238         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1239         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1240             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1241             if mapUser and len(mapUser[0]) == 3:
1242                 user = mapUser[0][0]
1243                 fullname = mapUser[0][1]
1244                 email = mapUser[0][2]
1245                 self.users[user] = fullname + " <" + email + ">"
1246                 self.emails[email] = user
1247
1248         s = ''
1249         for (key, val) in self.users.items():
1250             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1251
1252         open(self.getUserCacheFilename(), "wb").write(s)
1253         self.userMapFromPerforceServer = True
1254
1255     def loadUserMapFromCache(self):
1256         self.users = {}
1257         self.userMapFromPerforceServer = False
1258         try:
1259             cache = open(self.getUserCacheFilename(), "rb")
1260             lines = cache.readlines()
1261             cache.close()
1262             for line in lines:
1263                 entry = line.strip().split("\t")
1264                 self.users[entry[0]] = entry[1]
1265         except IOError:
1266             self.getUserMapFromPerforceServer()
1267
1268 class P4Debug(Command):
1269     def __init__(self):
1270         Command.__init__(self)
1271         self.options = []
1272         self.description = "A tool to debug the output of p4 -G."
1273         self.needsGit = False
1274
1275     def run(self, args):
1276         j = 0
1277         for output in p4CmdList(args):
1278             print 'Element: %d' % j
1279             j += 1
1280             print output
1281         return True
1282
1283 class P4RollBack(Command):
1284     def __init__(self):
1285         Command.__init__(self)
1286         self.options = [
1287             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1288         ]
1289         self.description = "A tool to debug the multi-branch import. Don't use :)"
1290         self.rollbackLocalBranches = False
1291
1292     def run(self, args):
1293         if len(args) != 1:
1294             return False
1295         maxChange = int(args[0])
1296
1297         if "p4ExitCode" in p4Cmd("changes -m 1"):
1298             die("Problems executing p4");
1299
1300         if self.rollbackLocalBranches:
1301             refPrefix = "refs/heads/"
1302             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1303         else:
1304             refPrefix = "refs/remotes/"
1305             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1306
1307         for line in lines:
1308             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1309                 line = line.strip()
1310                 ref = refPrefix + line
1311                 log = extractLogMessageFromGitCommit(ref)
1312                 settings = extractSettingsGitLog(log)
1313
1314                 depotPaths = settings['depot-paths']
1315                 change = settings['change']
1316
1317                 changed = False
1318
1319                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1320                                                            for p in depotPaths]))) == 0:
1321                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1322                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1323                     continue
1324
1325                 while change and int(change) > maxChange:
1326                     changed = True
1327                     if self.verbose:
1328                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1329                     system("git update-ref %s \"%s^\"" % (ref, ref))
1330                     log = extractLogMessageFromGitCommit(ref)
1331                     settings =  extractSettingsGitLog(log)
1332
1333
1334                     depotPaths = settings['depot-paths']
1335                     change = settings['change']
1336
1337                 if changed:
1338                     print "%s rewound to %s" % (ref, change)
1339
1340         return True
1341
1342 class P4Submit(Command, P4UserMap):
1343
1344     conflict_behavior_choices = ("ask", "skip", "quit")
1345
1346     def __init__(self):
1347         Command.__init__(self)
1348         P4UserMap.__init__(self)
1349         self.options = [
1350                 optparse.make_option("--origin", dest="origin"),
1351                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1352                 # preserve the user, requires relevant p4 permissions
1353                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1354                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1355                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1356                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1357                 optparse.make_option("--conflict", dest="conflict_behavior",
1358                                      choices=self.conflict_behavior_choices),
1359                 optparse.make_option("--branch", dest="branch"),
1360                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1361                                      help="Shelve instead of submit. Shelved files are reverted, "
1362                                      "restoring the workspace to the state before the shelve"),
1363                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1364                                      metavar="CHANGELIST",
1365                                      help="update an existing shelved changelist, implies --shelve, "
1366                                            "repeat in-order for multiple shelved changelists")
1367         ]
1368         self.description = "Submit changes from git to the perforce depot."
1369         self.usage += " [name of git branch to submit into perforce depot]"
1370         self.origin = ""
1371         self.detectRenames = False
1372         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1373         self.dry_run = False
1374         self.shelve = False
1375         self.update_shelve = list()
1376         self.prepare_p4_only = False
1377         self.conflict_behavior = None
1378         self.isWindows = (platform.system() == "Windows")
1379         self.exportLabels = False
1380         self.p4HasMoveCommand = p4_has_move_command()
1381         self.branch = None
1382
1383         if gitConfig('git-p4.largeFileSystem'):
1384             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1385
1386     def check(self):
1387         if len(p4CmdList("opened ...")) > 0:
1388             die("You have files opened with perforce! Close them before starting the sync.")
1389
1390     def separate_jobs_from_description(self, message):
1391         """Extract and return a possible Jobs field in the commit
1392            message.  It goes into a separate section in the p4 change
1393            specification.
1394
1395            A jobs line starts with "Jobs:" and looks like a new field
1396            in a form.  Values are white-space separated on the same
1397            line or on following lines that start with a tab.
1398
1399            This does not parse and extract the full git commit message
1400            like a p4 form.  It just sees the Jobs: line as a marker
1401            to pass everything from then on directly into the p4 form,
1402            but outside the description section.
1403
1404            Return a tuple (stripped log message, jobs string)."""
1405
1406         m = re.search(r'^Jobs:', message, re.MULTILINE)
1407         if m is None:
1408             return (message, None)
1409
1410         jobtext = message[m.start():]
1411         stripped_message = message[:m.start()].rstrip()
1412         return (stripped_message, jobtext)
1413
1414     def prepareLogMessage(self, template, message, jobs):
1415         """Edits the template returned from "p4 change -o" to insert
1416            the message in the Description field, and the jobs text in
1417            the Jobs field."""
1418         result = ""
1419
1420         inDescriptionSection = False
1421
1422         for line in template.split("\n"):
1423             if line.startswith("#"):
1424                 result += line + "\n"
1425                 continue
1426
1427             if inDescriptionSection:
1428                 if line.startswith("Files:") or line.startswith("Jobs:"):
1429                     inDescriptionSection = False
1430                     # insert Jobs section
1431                     if jobs:
1432                         result += jobs + "\n"
1433                 else:
1434                     continue
1435             else:
1436                 if line.startswith("Description:"):
1437                     inDescriptionSection = True
1438                     line += "\n"
1439                     for messageLine in message.split("\n"):
1440                         line += "\t" + messageLine + "\n"
1441
1442             result += line + "\n"
1443
1444         return result
1445
1446     def patchRCSKeywords(self, file, pattern):
1447         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1448         (handle, outFileName) = tempfile.mkstemp(dir='.')
1449         try:
1450             outFile = os.fdopen(handle, "w+")
1451             inFile = open(file, "r")
1452             regexp = re.compile(pattern, re.VERBOSE)
1453             for line in inFile.readlines():
1454                 line = regexp.sub(r'$\1$', line)
1455                 outFile.write(line)
1456             inFile.close()
1457             outFile.close()
1458             # Forcibly overwrite the original file
1459             os.unlink(file)
1460             shutil.move(outFileName, file)
1461         except:
1462             # cleanup our temporary file
1463             os.unlink(outFileName)
1464             print "Failed to strip RCS keywords in %s" % file
1465             raise
1466
1467         print "Patched up RCS keywords in %s" % file
1468
1469     def p4UserForCommit(self,id):
1470         # Return the tuple (perforce user,git email) for a given git commit id
1471         self.getUserMapFromPerforceServer()
1472         gitEmail = read_pipe(["git", "log", "--max-count=1",
1473                               "--format=%ae", id])
1474         gitEmail = gitEmail.strip()
1475         if not self.emails.has_key(gitEmail):
1476             return (None,gitEmail)
1477         else:
1478             return (self.emails[gitEmail],gitEmail)
1479
1480     def checkValidP4Users(self,commits):
1481         # check if any git authors cannot be mapped to p4 users
1482         for id in commits:
1483             (user,email) = self.p4UserForCommit(id)
1484             if not user:
1485                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1486                 if gitConfigBool("git-p4.allowMissingP4Users"):
1487                     print "%s" % msg
1488                 else:
1489                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1490
1491     def lastP4Changelist(self):
1492         # Get back the last changelist number submitted in this client spec. This
1493         # then gets used to patch up the username in the change. If the same
1494         # client spec is being used by multiple processes then this might go
1495         # wrong.
1496         results = p4CmdList("client -o")        # find the current client
1497         client = None
1498         for r in results:
1499             if r.has_key('Client'):
1500                 client = r['Client']
1501                 break
1502         if not client:
1503             die("could not get client spec")
1504         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1505         for r in results:
1506             if r.has_key('change'):
1507                 return r['change']
1508         die("Could not get changelist number for last submit - cannot patch up user details")
1509
1510     def modifyChangelistUser(self, changelist, newUser):
1511         # fixup the user field of a changelist after it has been submitted.
1512         changes = p4CmdList("change -o %s" % changelist)
1513         if len(changes) != 1:
1514             die("Bad output from p4 change modifying %s to user %s" %
1515                 (changelist, newUser))
1516
1517         c = changes[0]
1518         if c['User'] == newUser: return   # nothing to do
1519         c['User'] = newUser
1520         input = marshal.dumps(c)
1521
1522         result = p4CmdList("change -f -i", stdin=input)
1523         for r in result:
1524             if r.has_key('code'):
1525                 if r['code'] == 'error':
1526                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1527             if r.has_key('data'):
1528                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1529                 return
1530         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1531
1532     def canChangeChangelists(self):
1533         # check to see if we have p4 admin or super-user permissions, either of
1534         # which are required to modify changelists.
1535         results = p4CmdList(["protects", self.depotPath])
1536         for r in results:
1537             if r.has_key('perm'):
1538                 if r['perm'] == 'admin':
1539                     return 1
1540                 if r['perm'] == 'super':
1541                     return 1
1542         return 0
1543
1544     def prepareSubmitTemplate(self, changelist=None):
1545         """Run "p4 change -o" to grab a change specification template.
1546            This does not use "p4 -G", as it is nice to keep the submission
1547            template in original order, since a human might edit it.
1548
1549            Remove lines in the Files section that show changes to files
1550            outside the depot path we're committing into."""
1551
1552         [upstream, settings] = findUpstreamBranchPoint()
1553
1554         template = """\
1555 # A Perforce Change Specification.
1556 #
1557 #  Change:      The change number. 'new' on a new changelist.
1558 #  Date:        The date this specification was last modified.
1559 #  Client:      The client on which the changelist was created.  Read-only.
1560 #  User:        The user who created the changelist.
1561 #  Status:      Either 'pending' or 'submitted'. Read-only.
1562 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1563 #  Description: Comments about the changelist.  Required.
1564 #  Jobs:        What opened jobs are to be closed by this changelist.
1565 #               You may delete jobs from this list.  (New changelists only.)
1566 #  Files:       What opened files from the default changelist are to be added
1567 #               to this changelist.  You may delete files from this list.
1568 #               (New changelists only.)
1569 """
1570         files_list = []
1571         inFilesSection = False
1572         change_entry = None
1573         args = ['change', '-o']
1574         if changelist:
1575             args.append(str(changelist))
1576         for entry in p4CmdList(args):
1577             if not entry.has_key('code'):
1578                 continue
1579             if entry['code'] == 'stat':
1580                 change_entry = entry
1581                 break
1582         if not change_entry:
1583             die('Failed to decode output of p4 change -o')
1584         for key, value in change_entry.iteritems():
1585             if key.startswith('File'):
1586                 if settings.has_key('depot-paths'):
1587                     if not [p for p in settings['depot-paths']
1588                             if p4PathStartsWith(value, p)]:
1589                         continue
1590                 else:
1591                     if not p4PathStartsWith(value, self.depotPath):
1592                         continue
1593                 files_list.append(value)
1594                 continue
1595         # Output in the order expected by prepareLogMessage
1596         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1597             if not change_entry.has_key(key):
1598                 continue
1599             template += '\n'
1600             template += key + ':'
1601             if key == 'Description':
1602                 template += '\n'
1603             for field_line in change_entry[key].splitlines():
1604                 template += '\t'+field_line+'\n'
1605         if len(files_list) > 0:
1606             template += '\n'
1607             template += 'Files:\n'
1608         for path in files_list:
1609             template += '\t'+path+'\n'
1610         return template
1611
1612     def edit_template(self, template_file):
1613         """Invoke the editor to let the user change the submission
1614            message.  Return true if okay to continue with the submit."""
1615
1616         # if configured to skip the editing part, just submit
1617         if gitConfigBool("git-p4.skipSubmitEdit"):
1618             return True
1619
1620         # look at the modification time, to check later if the user saved
1621         # the file
1622         mtime = os.stat(template_file).st_mtime
1623
1624         # invoke the editor
1625         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1626             editor = os.environ.get("P4EDITOR")
1627         else:
1628             editor = read_pipe("git var GIT_EDITOR").strip()
1629         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1630
1631         # If the file was not saved, prompt to see if this patch should
1632         # be skipped.  But skip this verification step if configured so.
1633         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1634             return True
1635
1636         # modification time updated means user saved the file
1637         if os.stat(template_file).st_mtime > mtime:
1638             return True
1639
1640         while True:
1641             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1642             if response == 'y':
1643                 return True
1644             if response == 'n':
1645                 return False
1646
1647     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1648         # diff
1649         if os.environ.has_key("P4DIFF"):
1650             del(os.environ["P4DIFF"])
1651         diff = ""
1652         for editedFile in editedFiles:
1653             diff += p4_read_pipe(['diff', '-du',
1654                                   wildcard_encode(editedFile)])
1655
1656         # new file diff
1657         newdiff = ""
1658         for newFile in filesToAdd:
1659             newdiff += "==== new file ====\n"
1660             newdiff += "--- /dev/null\n"
1661             newdiff += "+++ %s\n" % newFile
1662
1663             is_link = os.path.islink(newFile)
1664             expect_link = newFile in symlinks
1665
1666             if is_link and expect_link:
1667                 newdiff += "+%s\n" % os.readlink(newFile)
1668             else:
1669                 f = open(newFile, "r")
1670                 for line in f.readlines():
1671                     newdiff += "+" + line
1672                 f.close()
1673
1674         return (diff + newdiff).replace('\r\n', '\n')
1675
1676     def applyCommit(self, id):
1677         """Apply one commit, return True if it succeeded."""
1678
1679         print "Applying", read_pipe(["git", "show", "-s",
1680                                      "--format=format:%h %s", id])
1681
1682         (p4User, gitEmail) = self.p4UserForCommit(id)
1683
1684         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1685         filesToAdd = set()
1686         filesToChangeType = set()
1687         filesToDelete = set()
1688         editedFiles = set()
1689         pureRenameCopy = set()
1690         symlinks = set()
1691         filesToChangeExecBit = {}
1692         all_files = list()
1693
1694         for line in diff:
1695             diff = parseDiffTreeEntry(line)
1696             modifier = diff['status']
1697             path = diff['src']
1698             all_files.append(path)
1699
1700             if modifier == "M":
1701                 p4_edit(path)
1702                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1703                     filesToChangeExecBit[path] = diff['dst_mode']
1704                 editedFiles.add(path)
1705             elif modifier == "A":
1706                 filesToAdd.add(path)
1707                 filesToChangeExecBit[path] = diff['dst_mode']
1708                 if path in filesToDelete:
1709                     filesToDelete.remove(path)
1710
1711                 dst_mode = int(diff['dst_mode'], 8)
1712                 if dst_mode == 0120000:
1713                     symlinks.add(path)
1714
1715             elif modifier == "D":
1716                 filesToDelete.add(path)
1717                 if path in filesToAdd:
1718                     filesToAdd.remove(path)
1719             elif modifier == "C":
1720                 src, dest = diff['src'], diff['dst']
1721                 p4_integrate(src, dest)
1722                 pureRenameCopy.add(dest)
1723                 if diff['src_sha1'] != diff['dst_sha1']:
1724                     p4_edit(dest)
1725                     pureRenameCopy.discard(dest)
1726                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1727                     p4_edit(dest)
1728                     pureRenameCopy.discard(dest)
1729                     filesToChangeExecBit[dest] = diff['dst_mode']
1730                 if self.isWindows:
1731                     # turn off read-only attribute
1732                     os.chmod(dest, stat.S_IWRITE)
1733                 os.unlink(dest)
1734                 editedFiles.add(dest)
1735             elif modifier == "R":
1736                 src, dest = diff['src'], diff['dst']
1737                 if self.p4HasMoveCommand:
1738                     p4_edit(src)        # src must be open before move
1739                     p4_move(src, dest)  # opens for (move/delete, move/add)
1740                 else:
1741                     p4_integrate(src, dest)
1742                     if diff['src_sha1'] != diff['dst_sha1']:
1743                         p4_edit(dest)
1744                     else:
1745                         pureRenameCopy.add(dest)
1746                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1747                     if not self.p4HasMoveCommand:
1748                         p4_edit(dest)   # with move: already open, writable
1749                     filesToChangeExecBit[dest] = diff['dst_mode']
1750                 if not self.p4HasMoveCommand:
1751                     if self.isWindows:
1752                         os.chmod(dest, stat.S_IWRITE)
1753                     os.unlink(dest)
1754                     filesToDelete.add(src)
1755                 editedFiles.add(dest)
1756             elif modifier == "T":
1757                 filesToChangeType.add(path)
1758             else:
1759                 die("unknown modifier %s for %s" % (modifier, path))
1760
1761         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1762         patchcmd = diffcmd + " | git apply "
1763         tryPatchCmd = patchcmd + "--check -"
1764         applyPatchCmd = patchcmd + "--check --apply -"
1765         patch_succeeded = True
1766
1767         if os.system(tryPatchCmd) != 0:
1768             fixed_rcs_keywords = False
1769             patch_succeeded = False
1770             print "Unfortunately applying the change failed!"
1771
1772             # Patch failed, maybe it's just RCS keyword woes. Look through
1773             # the patch to see if that's possible.
1774             if gitConfigBool("git-p4.attemptRCSCleanup"):
1775                 file = None
1776                 pattern = None
1777                 kwfiles = {}
1778                 for file in editedFiles | filesToDelete:
1779                     # did this file's delta contain RCS keywords?
1780                     pattern = p4_keywords_regexp_for_file(file)
1781
1782                     if pattern:
1783                         # this file is a possibility...look for RCS keywords.
1784                         regexp = re.compile(pattern, re.VERBOSE)
1785                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1786                             if regexp.search(line):
1787                                 if verbose:
1788                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1789                                 kwfiles[file] = pattern
1790                                 break
1791
1792                 for file in kwfiles:
1793                     if verbose:
1794                         print "zapping %s with %s" % (line,pattern)
1795                     # File is being deleted, so not open in p4.  Must
1796                     # disable the read-only bit on windows.
1797                     if self.isWindows and file not in editedFiles:
1798                         os.chmod(file, stat.S_IWRITE)
1799                     self.patchRCSKeywords(file, kwfiles[file])
1800                     fixed_rcs_keywords = True
1801
1802             if fixed_rcs_keywords:
1803                 print "Retrying the patch with RCS keywords cleaned up"
1804                 if os.system(tryPatchCmd) == 0:
1805                     patch_succeeded = True
1806
1807         if not patch_succeeded:
1808             for f in editedFiles:
1809                 p4_revert(f)
1810             return False
1811
1812         #
1813         # Apply the patch for real, and do add/delete/+x handling.
1814         #
1815         system(applyPatchCmd)
1816
1817         for f in filesToChangeType:
1818             p4_edit(f, "-t", "auto")
1819         for f in filesToAdd:
1820             p4_add(f)
1821         for f in filesToDelete:
1822             p4_revert(f)
1823             p4_delete(f)
1824
1825         # Set/clear executable bits
1826         for f in filesToChangeExecBit.keys():
1827             mode = filesToChangeExecBit[f]
1828             setP4ExecBit(f, mode)
1829
1830         update_shelve = 0
1831         if len(self.update_shelve) > 0:
1832             update_shelve = self.update_shelve.pop(0)
1833             p4_reopen_in_change(update_shelve, all_files)
1834
1835         #
1836         # Build p4 change description, starting with the contents
1837         # of the git commit message.
1838         #
1839         logMessage = extractLogMessageFromGitCommit(id)
1840         logMessage = logMessage.strip()
1841         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1842
1843         template = self.prepareSubmitTemplate(update_shelve)
1844         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1845
1846         if self.preserveUser:
1847            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1848
1849         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1850             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1851             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1852             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1853
1854         separatorLine = "######## everything below this line is just the diff #######\n"
1855         if not self.prepare_p4_only:
1856             submitTemplate += separatorLine
1857             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1858
1859         (handle, fileName) = tempfile.mkstemp()
1860         tmpFile = os.fdopen(handle, "w+b")
1861         if self.isWindows:
1862             submitTemplate = submitTemplate.replace("\n", "\r\n")
1863         tmpFile.write(submitTemplate)
1864         tmpFile.close()
1865
1866         if self.prepare_p4_only:
1867             #
1868             # Leave the p4 tree prepared, and the submit template around
1869             # and let the user decide what to do next
1870             #
1871             print
1872             print "P4 workspace prepared for submission."
1873             print "To submit or revert, go to client workspace"
1874             print "  " + self.clientPath
1875             print
1876             print "To submit, use \"p4 submit\" to write a new description,"
1877             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1878                   " \"git p4\"." % fileName
1879             print "You can delete the file \"%s\" when finished." % fileName
1880
1881             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1882                 print "To preserve change ownership by user %s, you must\n" \
1883                       "do \"p4 change -f <change>\" after submitting and\n" \
1884                       "edit the User field."
1885             if pureRenameCopy:
1886                 print "After submitting, renamed files must be re-synced."
1887                 print "Invoke \"p4 sync -f\" on each of these files:"
1888                 for f in pureRenameCopy:
1889                     print "  " + f
1890
1891             print
1892             print "To revert the changes, use \"p4 revert ...\", and delete"
1893             print "the submit template file \"%s\"" % fileName
1894             if filesToAdd:
1895                 print "Since the commit adds new files, they must be deleted:"
1896                 for f in filesToAdd:
1897                     print "  " + f
1898             print
1899             return True
1900
1901         #
1902         # Let the user edit the change description, then submit it.
1903         #
1904         submitted = False
1905
1906         try:
1907             if self.edit_template(fileName):
1908                 # read the edited message and submit
1909                 tmpFile = open(fileName, "rb")
1910                 message = tmpFile.read()
1911                 tmpFile.close()
1912                 if self.isWindows:
1913                     message = message.replace("\r\n", "\n")
1914                 submitTemplate = message[:message.index(separatorLine)]
1915
1916                 if update_shelve:
1917                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1918                 elif self.shelve:
1919                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1920                 else:
1921                     p4_write_pipe(['submit', '-i'], submitTemplate)
1922                     # The rename/copy happened by applying a patch that created a
1923                     # new file.  This leaves it writable, which confuses p4.
1924                     for f in pureRenameCopy:
1925                         p4_sync(f, "-f")
1926
1927                 if self.preserveUser:
1928                     if p4User:
1929                         # Get last changelist number. Cannot easily get it from
1930                         # the submit command output as the output is
1931                         # unmarshalled.
1932                         changelist = self.lastP4Changelist()
1933                         self.modifyChangelistUser(changelist, p4User)
1934
1935                 submitted = True
1936
1937         finally:
1938             # skip this patch
1939             if not submitted or self.shelve:
1940                 if self.shelve:
1941                     print ("Reverting shelved files.")
1942                 else:
1943                     print ("Submission cancelled, undoing p4 changes.")
1944                 for f in editedFiles | filesToDelete:
1945                     p4_revert(f)
1946                 for f in filesToAdd:
1947                     p4_revert(f)
1948                     os.remove(f)
1949
1950         os.remove(fileName)
1951         return submitted
1952
1953     # Export git tags as p4 labels. Create a p4 label and then tag
1954     # with that.
1955     def exportGitTags(self, gitTags):
1956         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1957         if len(validLabelRegexp) == 0:
1958             validLabelRegexp = defaultLabelRegexp
1959         m = re.compile(validLabelRegexp)
1960
1961         for name in gitTags:
1962
1963             if not m.match(name):
1964                 if verbose:
1965                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1966                 continue
1967
1968             # Get the p4 commit this corresponds to
1969             logMessage = extractLogMessageFromGitCommit(name)
1970             values = extractSettingsGitLog(logMessage)
1971
1972             if not values.has_key('change'):
1973                 # a tag pointing to something not sent to p4; ignore
1974                 if verbose:
1975                     print "git tag %s does not give a p4 commit" % name
1976                 continue
1977             else:
1978                 changelist = values['change']
1979
1980             # Get the tag details.
1981             inHeader = True
1982             isAnnotated = False
1983             body = []
1984             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1985                 l = l.strip()
1986                 if inHeader:
1987                     if re.match(r'tag\s+', l):
1988                         isAnnotated = True
1989                     elif re.match(r'\s*$', l):
1990                         inHeader = False
1991                         continue
1992                 else:
1993                     body.append(l)
1994
1995             if not isAnnotated:
1996                 body = ["lightweight tag imported by git p4\n"]
1997
1998             # Create the label - use the same view as the client spec we are using
1999             clientSpec = getClientSpec()
2000
2001             labelTemplate  = "Label: %s\n" % name
2002             labelTemplate += "Description:\n"
2003             for b in body:
2004                 labelTemplate += "\t" + b + "\n"
2005             labelTemplate += "View:\n"
2006             for depot_side in clientSpec.mappings:
2007                 labelTemplate += "\t%s\n" % depot_side
2008
2009             if self.dry_run:
2010                 print "Would create p4 label %s for tag" % name
2011             elif self.prepare_p4_only:
2012                 print "Not creating p4 label %s for tag due to option" \
2013                       " --prepare-p4-only" % name
2014             else:
2015                 p4_write_pipe(["label", "-i"], labelTemplate)
2016
2017                 # Use the label
2018                 p4_system(["tag", "-l", name] +
2019                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2020
2021                 if verbose:
2022                     print "created p4 label for tag %s" % name
2023
2024     def run(self, args):
2025         if len(args) == 0:
2026             self.master = currentGitBranch()
2027         elif len(args) == 1:
2028             self.master = args[0]
2029             if not branchExists(self.master):
2030                 die("Branch %s does not exist" % self.master)
2031         else:
2032             return False
2033
2034         for i in self.update_shelve:
2035             if i <= 0:
2036                 sys.exit("invalid changelist %d" % i)
2037
2038         if self.master:
2039             allowSubmit = gitConfig("git-p4.allowSubmit")
2040             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2041                 die("%s is not in git-p4.allowSubmit" % self.master)
2042
2043         [upstream, settings] = findUpstreamBranchPoint()
2044         self.depotPath = settings['depot-paths'][0]
2045         if len(self.origin) == 0:
2046             self.origin = upstream
2047
2048         if len(self.update_shelve) > 0:
2049             self.shelve = True
2050
2051         if self.preserveUser:
2052             if not self.canChangeChangelists():
2053                 die("Cannot preserve user names without p4 super-user or admin permissions")
2054
2055         # if not set from the command line, try the config file
2056         if self.conflict_behavior is None:
2057             val = gitConfig("git-p4.conflict")
2058             if val:
2059                 if val not in self.conflict_behavior_choices:
2060                     die("Invalid value '%s' for config git-p4.conflict" % val)
2061             else:
2062                 val = "ask"
2063             self.conflict_behavior = val
2064
2065         if self.verbose:
2066             print "Origin branch is " + self.origin
2067
2068         if len(self.depotPath) == 0:
2069             print "Internal error: cannot locate perforce depot path from existing branches"
2070             sys.exit(128)
2071
2072         self.useClientSpec = False
2073         if gitConfigBool("git-p4.useclientspec"):
2074             self.useClientSpec = True
2075         if self.useClientSpec:
2076             self.clientSpecDirs = getClientSpec()
2077
2078         # Check for the existence of P4 branches
2079         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2080
2081         if self.useClientSpec and not branchesDetected:
2082             # all files are relative to the client spec
2083             self.clientPath = getClientRoot()
2084         else:
2085             self.clientPath = p4Where(self.depotPath)
2086
2087         if self.clientPath == "":
2088             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2089
2090         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2091         self.oldWorkingDirectory = os.getcwd()
2092
2093         # ensure the clientPath exists
2094         new_client_dir = False
2095         if not os.path.exists(self.clientPath):
2096             new_client_dir = True
2097             os.makedirs(self.clientPath)
2098
2099         chdir(self.clientPath, is_client_path=True)
2100         if self.dry_run:
2101             print "Would synchronize p4 checkout in %s" % self.clientPath
2102         else:
2103             print "Synchronizing p4 checkout..."
2104             if new_client_dir:
2105                 # old one was destroyed, and maybe nobody told p4
2106                 p4_sync("...", "-f")
2107             else:
2108                 p4_sync("...")
2109         self.check()
2110
2111         commits = []
2112         if self.master:
2113             committish = self.master
2114         else:
2115             committish = 'HEAD'
2116
2117         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2118             commits.append(line.strip())
2119         commits.reverse()
2120
2121         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2122             self.checkAuthorship = False
2123         else:
2124             self.checkAuthorship = True
2125
2126         if self.preserveUser:
2127             self.checkValidP4Users(commits)
2128
2129         #
2130         # Build up a set of options to be passed to diff when
2131         # submitting each commit to p4.
2132         #
2133         if self.detectRenames:
2134             # command-line -M arg
2135             self.diffOpts = "-M"
2136         else:
2137             # If not explicitly set check the config variable
2138             detectRenames = gitConfig("git-p4.detectRenames")
2139
2140             if detectRenames.lower() == "false" or detectRenames == "":
2141                 self.diffOpts = ""
2142             elif detectRenames.lower() == "true":
2143                 self.diffOpts = "-M"
2144             else:
2145                 self.diffOpts = "-M%s" % detectRenames
2146
2147         # no command-line arg for -C or --find-copies-harder, just
2148         # config variables
2149         detectCopies = gitConfig("git-p4.detectCopies")
2150         if detectCopies.lower() == "false" or detectCopies == "":
2151             pass
2152         elif detectCopies.lower() == "true":
2153             self.diffOpts += " -C"
2154         else:
2155             self.diffOpts += " -C%s" % detectCopies
2156
2157         if gitConfigBool("git-p4.detectCopiesHarder"):
2158             self.diffOpts += " --find-copies-harder"
2159
2160         num_shelves = len(self.update_shelve)
2161         if num_shelves > 0 and num_shelves != len(commits):
2162             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2163                      (len(commits), num_shelves))
2164
2165         #
2166         # Apply the commits, one at a time.  On failure, ask if should
2167         # continue to try the rest of the patches, or quit.
2168         #
2169         if self.dry_run:
2170             print "Would apply"
2171         applied = []
2172         last = len(commits) - 1
2173         for i, commit in enumerate(commits):
2174             if self.dry_run:
2175                 print " ", read_pipe(["git", "show", "-s",
2176                                       "--format=format:%h %s", commit])
2177                 ok = True
2178             else:
2179                 ok = self.applyCommit(commit)
2180             if ok:
2181                 applied.append(commit)
2182             else:
2183                 if self.prepare_p4_only and i < last:
2184                     print "Processing only the first commit due to option" \
2185                           " --prepare-p4-only"
2186                     break
2187                 if i < last:
2188                     quit = False
2189                     while True:
2190                         # prompt for what to do, or use the option/variable
2191                         if self.conflict_behavior == "ask":
2192                             print "What do you want to do?"
2193                             response = raw_input("[s]kip this commit but apply"
2194                                                  " the rest, or [q]uit? ")
2195                             if not response:
2196                                 continue
2197                         elif self.conflict_behavior == "skip":
2198                             response = "s"
2199                         elif self.conflict_behavior == "quit":
2200                             response = "q"
2201                         else:
2202                             die("Unknown conflict_behavior '%s'" %
2203                                 self.conflict_behavior)
2204
2205                         if response[0] == "s":
2206                             print "Skipping this commit, but applying the rest"
2207                             break
2208                         if response[0] == "q":
2209                             print "Quitting"
2210                             quit = True
2211                             break
2212                     if quit:
2213                         break
2214
2215         chdir(self.oldWorkingDirectory)
2216         shelved_applied = "shelved" if self.shelve else "applied"
2217         if self.dry_run:
2218             pass
2219         elif self.prepare_p4_only:
2220             pass
2221         elif len(commits) == len(applied):
2222             print ("All commits {0}!".format(shelved_applied))
2223
2224             sync = P4Sync()
2225             if self.branch:
2226                 sync.branch = self.branch
2227             sync.run([])
2228
2229             rebase = P4Rebase()
2230             rebase.rebase()
2231
2232         else:
2233             if len(applied) == 0:
2234                 print ("No commits {0}.".format(shelved_applied))
2235             else:
2236                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2237                 for c in commits:
2238                     if c in applied:
2239                         star = "*"
2240                     else:
2241                         star = " "
2242                     print star, read_pipe(["git", "show", "-s",
2243                                            "--format=format:%h %s",  c])
2244                 print "You will have to do 'git p4 sync' and rebase."
2245
2246         if gitConfigBool("git-p4.exportLabels"):
2247             self.exportLabels = True
2248
2249         if self.exportLabels:
2250             p4Labels = getP4Labels(self.depotPath)
2251             gitTags = getGitTags()
2252
2253             missingGitTags = gitTags - p4Labels
2254             self.exportGitTags(missingGitTags)
2255
2256         # exit with error unless everything applied perfectly
2257         if len(commits) != len(applied):
2258                 sys.exit(1)
2259
2260         return True
2261
2262 class View(object):
2263     """Represent a p4 view ("p4 help views"), and map files in a
2264        repo according to the view."""
2265
2266     def __init__(self, client_name):
2267         self.mappings = []
2268         self.client_prefix = "//%s/" % client_name
2269         # cache results of "p4 where" to lookup client file locations
2270         self.client_spec_path_cache = {}
2271
2272     def append(self, view_line):
2273         """Parse a view line, splitting it into depot and client
2274            sides.  Append to self.mappings, preserving order.  This
2275            is only needed for tag creation."""
2276
2277         # Split the view line into exactly two words.  P4 enforces
2278         # structure on these lines that simplifies this quite a bit.
2279         #
2280         # Either or both words may be double-quoted.
2281         # Single quotes do not matter.
2282         # Double-quote marks cannot occur inside the words.
2283         # A + or - prefix is also inside the quotes.
2284         # There are no quotes unless they contain a space.
2285         # The line is already white-space stripped.
2286         # The two words are separated by a single space.
2287         #
2288         if view_line[0] == '"':
2289             # First word is double quoted.  Find its end.
2290             close_quote_index = view_line.find('"', 1)
2291             if close_quote_index <= 0:
2292                 die("No first-word closing quote found: %s" % view_line)
2293             depot_side = view_line[1:close_quote_index]
2294             # skip closing quote and space
2295             rhs_index = close_quote_index + 1 + 1
2296         else:
2297             space_index = view_line.find(" ")
2298             if space_index <= 0:
2299                 die("No word-splitting space found: %s" % view_line)
2300             depot_side = view_line[0:space_index]
2301             rhs_index = space_index + 1
2302
2303         # prefix + means overlay on previous mapping
2304         if depot_side.startswith("+"):
2305             depot_side = depot_side[1:]
2306
2307         # prefix - means exclude this path, leave out of mappings
2308         exclude = False
2309         if depot_side.startswith("-"):
2310             exclude = True
2311             depot_side = depot_side[1:]
2312
2313         if not exclude:
2314             self.mappings.append(depot_side)
2315
2316     def convert_client_path(self, clientFile):
2317         # chop off //client/ part to make it relative
2318         if not clientFile.startswith(self.client_prefix):
2319             die("No prefix '%s' on clientFile '%s'" %
2320                 (self.client_prefix, clientFile))
2321         return clientFile[len(self.client_prefix):]
2322
2323     def update_client_spec_path_cache(self, files):
2324         """ Caching file paths by "p4 where" batch query """
2325
2326         # List depot file paths exclude that already cached
2327         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2328
2329         if len(fileArgs) == 0:
2330             return  # All files in cache
2331
2332         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2333         for res in where_result:
2334             if "code" in res and res["code"] == "error":
2335                 # assume error is "... file(s) not in client view"
2336                 continue
2337             if "clientFile" not in res:
2338                 die("No clientFile in 'p4 where' output")
2339             if "unmap" in res:
2340                 # it will list all of them, but only one not unmap-ped
2341                 continue
2342             if gitConfigBool("core.ignorecase"):
2343                 res['depotFile'] = res['depotFile'].lower()
2344             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2345
2346         # not found files or unmap files set to ""
2347         for depotFile in fileArgs:
2348             if gitConfigBool("core.ignorecase"):
2349                 depotFile = depotFile.lower()
2350             if depotFile not in self.client_spec_path_cache:
2351                 self.client_spec_path_cache[depotFile] = ""
2352
2353     def map_in_client(self, depot_path):
2354         """Return the relative location in the client where this
2355            depot file should live.  Returns "" if the file should
2356            not be mapped in the client."""
2357
2358         if gitConfigBool("core.ignorecase"):
2359             depot_path = depot_path.lower()
2360
2361         if depot_path in self.client_spec_path_cache:
2362             return self.client_spec_path_cache[depot_path]
2363
2364         die( "Error: %s is not found in client spec path" % depot_path )
2365         return ""
2366
2367 class P4Sync(Command, P4UserMap):
2368     delete_actions = ( "delete", "move/delete", "purge" )
2369
2370     def __init__(self):
2371         Command.__init__(self)
2372         P4UserMap.__init__(self)
2373         self.options = [
2374                 optparse.make_option("--branch", dest="branch"),
2375                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2376                 optparse.make_option("--changesfile", dest="changesFile"),
2377                 optparse.make_option("--silent", dest="silent", action="store_true"),
2378                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2379                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2380                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2381                                      help="Import into refs/heads/ , not refs/remotes"),
2382                 optparse.make_option("--max-changes", dest="maxChanges",
2383                                      help="Maximum number of changes to import"),
2384                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2385                                      help="Internal block size to use when iteratively calling p4 changes"),
2386                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2387                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2388                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2389                                      help="Only sync files that are included in the Perforce Client Spec"),
2390                 optparse.make_option("-/", dest="cloneExclude",
2391                                      action="append", type="string",
2392                                      help="exclude depot path"),
2393         ]
2394         self.description = """Imports from Perforce into a git repository.\n
2395     example:
2396     //depot/my/project/ -- to import the current head
2397     //depot/my/project/@all -- to import everything
2398     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2399
2400     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2401
2402         self.usage += " //depot/path[@revRange]"
2403         self.silent = False
2404         self.createdBranches = set()
2405         self.committedChanges = set()
2406         self.branch = ""
2407         self.detectBranches = False
2408         self.detectLabels = False
2409         self.importLabels = False
2410         self.changesFile = ""
2411         self.syncWithOrigin = True
2412         self.importIntoRemotes = True
2413         self.maxChanges = ""
2414         self.changes_block_size = None
2415         self.keepRepoPath = False
2416         self.depotPaths = None
2417         self.p4BranchesInGit = []
2418         self.cloneExclude = []
2419         self.useClientSpec = False
2420         self.useClientSpec_from_options = False
2421         self.clientSpecDirs = None
2422         self.tempBranches = []
2423         self.tempBranchLocation = "refs/git-p4-tmp"
2424         self.largeFileSystem = None
2425         self.suppress_meta_comment = False
2426
2427         if gitConfig('git-p4.largeFileSystem'):
2428             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2429             self.largeFileSystem = largeFileSystemConstructor(
2430                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2431             )
2432
2433         if gitConfig("git-p4.syncFromOrigin") == "false":
2434             self.syncWithOrigin = False
2435
2436         self.depotPaths = []
2437         self.changeRange = ""
2438         self.previousDepotPaths = []
2439         self.hasOrigin = False
2440
2441         # map from branch depot path to parent branch
2442         self.knownBranches = {}
2443         self.initialParents = {}
2444
2445         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2446         self.labels = {}
2447
2448     # Force a checkpoint in fast-import and wait for it to finish
2449     def checkpoint(self):
2450         self.gitStream.write("checkpoint\n\n")
2451         self.gitStream.write("progress checkpoint\n\n")
2452         out = self.gitOutput.readline()
2453         if self.verbose:
2454             print "checkpoint finished: " + out
2455
2456     def cmp_shelved(self, path, filerev, revision):
2457         """ Determine if a path at revision #filerev is the same as the file
2458             at revision @revision for a shelved changelist. If they don't match,
2459             unshelving won't be safe (we will get other changes mixed in).
2460
2461             This is comparing the revision that the shelved changelist is *based* on, not
2462             the shelved changelist itself.
2463         """
2464         ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2465         if verbose:
2466             print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2467         return ret["status"] == "identical"
2468
2469     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
2470         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2471                              for path in self.cloneExclude]
2472         files = []
2473         fnum = 0
2474         while commit.has_key("depotFile%s" % fnum):
2475             path =  commit["depotFile%s" % fnum]
2476
2477             if [p for p in self.cloneExclude
2478                 if p4PathStartsWith(path, p)]:
2479                 found = False
2480             else:
2481                 found = [p for p in self.depotPaths
2482                          if p4PathStartsWith(path, p)]
2483             if not found:
2484                 fnum = fnum + 1
2485                 continue
2486
2487             file = {}
2488             file["path"] = path
2489             file["rev"] = commit["rev%s" % fnum]
2490             file["action"] = commit["action%s" % fnum]
2491             file["type"] = commit["type%s" % fnum]
2492             if shelved:
2493                 file["shelved_cl"] = int(shelved_cl)
2494
2495                 # For shelved changelists, check that the revision of each file that the
2496                 # shelve was based on matches the revision that we are using for the
2497                 # starting point for git-fast-import (self.initialParent). Otherwise
2498                 # the resulting diff will contain deltas from multiple commits.
2499
2500                 if file["action"] != "add" and \
2501                     not self.cmp_shelved(path, file["rev"], origin_revision):
2502                     sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2503                         commit["change"], self.initialParent, path))
2504
2505             files.append(file)
2506             fnum = fnum + 1
2507         return files
2508
2509     def extractJobsFromCommit(self, commit):
2510         jobs = []
2511         jnum = 0
2512         while commit.has_key("job%s" % jnum):
2513             job = commit["job%s" % jnum]
2514             jobs.append(job)
2515             jnum = jnum + 1
2516         return jobs
2517
2518     def stripRepoPath(self, path, prefixes):
2519         """When streaming files, this is called to map a p4 depot path
2520            to where it should go in git.  The prefixes are either
2521            self.depotPaths, or self.branchPrefixes in the case of
2522            branch detection."""
2523
2524         if self.useClientSpec:
2525             # branch detection moves files up a level (the branch name)
2526             # from what client spec interpretation gives
2527             path = self.clientSpecDirs.map_in_client(path)
2528             if self.detectBranches:
2529                 for b in self.knownBranches:
2530                     if path.startswith(b + "/"):
2531                         path = path[len(b)+1:]
2532
2533         elif self.keepRepoPath:
2534             # Preserve everything in relative path name except leading
2535             # //depot/; just look at first prefix as they all should
2536             # be in the same depot.
2537             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2538             if p4PathStartsWith(path, depot):
2539                 path = path[len(depot):]
2540
2541         else:
2542             for p in prefixes:
2543                 if p4PathStartsWith(path, p):
2544                     path = path[len(p):]
2545                     break
2546
2547         path = wildcard_decode(path)
2548         return path
2549
2550     def splitFilesIntoBranches(self, commit):
2551         """Look at each depotFile in the commit to figure out to what
2552            branch it belongs."""
2553
2554         if self.clientSpecDirs:
2555             files = self.extractFilesFromCommit(commit)
2556             self.clientSpecDirs.update_client_spec_path_cache(files)
2557
2558         branches = {}
2559         fnum = 0
2560         while commit.has_key("depotFile%s" % fnum):
2561             path =  commit["depotFile%s" % fnum]
2562             found = [p for p in self.depotPaths
2563                      if p4PathStartsWith(path, p)]
2564             if not found:
2565                 fnum = fnum + 1
2566                 continue
2567
2568             file = {}
2569             file["path"] = path
2570             file["rev"] = commit["rev%s" % fnum]
2571             file["action"] = commit["action%s" % fnum]
2572             file["type"] = commit["type%s" % fnum]
2573             fnum = fnum + 1
2574
2575             # start with the full relative path where this file would
2576             # go in a p4 client
2577             if self.useClientSpec:
2578                 relPath = self.clientSpecDirs.map_in_client(path)
2579             else:
2580                 relPath = self.stripRepoPath(path, self.depotPaths)
2581
2582             for branch in self.knownBranches.keys():
2583                 # add a trailing slash so that a commit into qt/4.2foo
2584                 # doesn't end up in qt/4.2, e.g.
2585                 if relPath.startswith(branch + "/"):
2586                     if branch not in branches:
2587                         branches[branch] = []
2588                     branches[branch].append(file)
2589                     break
2590
2591         return branches
2592
2593     def writeToGitStream(self, gitMode, relPath, contents):
2594         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2595         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2596         for d in contents:
2597             self.gitStream.write(d)
2598         self.gitStream.write('\n')
2599
2600     def encodeWithUTF8(self, path):
2601         try:
2602             path.decode('ascii')
2603         except:
2604             encoding = 'utf8'
2605             if gitConfig('git-p4.pathEncoding'):
2606                 encoding = gitConfig('git-p4.pathEncoding')
2607             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2608             if self.verbose:
2609                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2610         return path
2611
2612     # output one file from the P4 stream
2613     # - helper for streamP4Files
2614
2615     def streamOneP4File(self, file, contents):
2616         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2617         relPath = self.encodeWithUTF8(relPath)
2618         if verbose:
2619             size = int(self.stream_file['fileSize'])
2620             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2621             sys.stdout.flush()
2622
2623         (type_base, type_mods) = split_p4_type(file["type"])
2624
2625         git_mode = "100644"
2626         if "x" in type_mods:
2627             git_mode = "100755"
2628         if type_base == "symlink":
2629             git_mode = "120000"
2630             # p4 print on a symlink sometimes contains "target\n";
2631             # if it does, remove the newline
2632             data = ''.join(contents)
2633             if not data:
2634                 # Some version of p4 allowed creating a symlink that pointed
2635                 # to nothing.  This causes p4 errors when checking out such
2636                 # a change, and errors here too.  Work around it by ignoring
2637                 # the bad symlink; hopefully a future change fixes it.
2638                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2639                 return
2640             elif data[-1] == '\n':
2641                 contents = [data[:-1]]
2642             else:
2643                 contents = [data]
2644
2645         if type_base == "utf16":
2646             # p4 delivers different text in the python output to -G
2647             # than it does when using "print -o", or normal p4 client
2648             # operations.  utf16 is converted to ascii or utf8, perhaps.
2649             # But ascii text saved as -t utf16 is completely mangled.
2650             # Invoke print -o to get the real contents.
2651             #
2652             # On windows, the newlines will always be mangled by print, so put
2653             # them back too.  This is not needed to the cygwin windows version,
2654             # just the native "NT" type.
2655             #
2656             try:
2657                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2658             except Exception as e:
2659                 if 'Translation of file content failed' in str(e):
2660                     type_base = 'binary'
2661                 else:
2662                     raise e
2663             else:
2664                 if p4_version_string().find('/NT') >= 0:
2665                     text = text.replace('\r\n', '\n')
2666                 contents = [ text ]
2667
2668         if type_base == "apple":
2669             # Apple filetype files will be streamed as a concatenation of
2670             # its appledouble header and the contents.  This is useless
2671             # on both macs and non-macs.  If using "print -q -o xx", it
2672             # will create "xx" with the data, and "%xx" with the header.
2673             # This is also not very useful.
2674             #
2675             # Ideally, someday, this script can learn how to generate
2676             # appledouble files directly and import those to git, but
2677             # non-mac machines can never find a use for apple filetype.
2678             print "\nIgnoring apple filetype file %s" % file['depotFile']
2679             return
2680
2681         # Note that we do not try to de-mangle keywords on utf16 files,
2682         # even though in theory somebody may want that.
2683         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2684         if pattern:
2685             regexp = re.compile(pattern, re.VERBOSE)
2686             text = ''.join(contents)
2687             text = regexp.sub(r'$\1$', text)
2688             contents = [ text ]
2689
2690         if self.largeFileSystem:
2691             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2692
2693         self.writeToGitStream(git_mode, relPath, contents)
2694
2695     def streamOneP4Deletion(self, file):
2696         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2697         relPath = self.encodeWithUTF8(relPath)
2698         if verbose:
2699             sys.stdout.write("delete %s\n" % relPath)
2700             sys.stdout.flush()
2701         self.gitStream.write("D %s\n" % relPath)
2702
2703         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2704             self.largeFileSystem.removeLargeFile(relPath)
2705
2706     # handle another chunk of streaming data
2707     def streamP4FilesCb(self, marshalled):
2708
2709         # catch p4 errors and complain
2710         err = None
2711         if "code" in marshalled:
2712             if marshalled["code"] == "error":
2713                 if "data" in marshalled:
2714                     err = marshalled["data"].rstrip()
2715
2716         if not err and 'fileSize' in self.stream_file:
2717             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2718             if required_bytes > 0:
2719                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2720                     os.getcwd(), required_bytes/1024/1024
2721                 )
2722
2723         if err:
2724             f = None
2725             if self.stream_have_file_info:
2726                 if "depotFile" in self.stream_file:
2727                     f = self.stream_file["depotFile"]
2728             # force a failure in fast-import, else an empty
2729             # commit will be made
2730             self.gitStream.write("\n")
2731             self.gitStream.write("die-now\n")
2732             self.gitStream.close()
2733             # ignore errors, but make sure it exits first
2734             self.importProcess.wait()
2735             if f:
2736                 die("Error from p4 print for %s: %s" % (f, err))
2737             else:
2738                 die("Error from p4 print: %s" % err)
2739
2740         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2741             # start of a new file - output the old one first
2742             self.streamOneP4File(self.stream_file, self.stream_contents)
2743             self.stream_file = {}
2744             self.stream_contents = []
2745             self.stream_have_file_info = False
2746
2747         # pick up the new file information... for the
2748         # 'data' field we need to append to our array
2749         for k in marshalled.keys():
2750             if k == 'data':
2751                 if 'streamContentSize' not in self.stream_file:
2752                     self.stream_file['streamContentSize'] = 0
2753                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2754                 self.stream_contents.append(marshalled['data'])
2755             else:
2756                 self.stream_file[k] = marshalled[k]
2757
2758         if (verbose and
2759             'streamContentSize' in self.stream_file and
2760             'fileSize' in self.stream_file and
2761             'depotFile' in self.stream_file):
2762             size = int(self.stream_file["fileSize"])
2763             if size > 0:
2764                 progress = 100*self.stream_file['streamContentSize']/size
2765                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2766                 sys.stdout.flush()
2767
2768         self.stream_have_file_info = True
2769
2770     # Stream directly from "p4 files" into "git fast-import"
2771     def streamP4Files(self, files):
2772         filesForCommit = []
2773         filesToRead = []
2774         filesToDelete = []
2775
2776         for f in files:
2777             filesForCommit.append(f)
2778             if f['action'] in self.delete_actions:
2779                 filesToDelete.append(f)
2780             else:
2781                 filesToRead.append(f)
2782
2783         # deleted files...
2784         for f in filesToDelete:
2785             self.streamOneP4Deletion(f)
2786
2787         if len(filesToRead) > 0:
2788             self.stream_file = {}
2789             self.stream_contents = []
2790             self.stream_have_file_info = False
2791
2792             # curry self argument
2793             def streamP4FilesCbSelf(entry):
2794                 self.streamP4FilesCb(entry)
2795
2796             fileArgs = []
2797             for f in filesToRead:
2798                 if 'shelved_cl' in f:
2799                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2800                     # the contents
2801                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2802                 else:
2803                     fileArg = '%s#%s' % (f['path'], f['rev'])
2804
2805                 fileArgs.append(fileArg)
2806
2807             p4CmdList(["-x", "-", "print"],
2808                       stdin=fileArgs,
2809                       cb=streamP4FilesCbSelf)
2810
2811             # do the last chunk
2812             if self.stream_file.has_key('depotFile'):
2813                 self.streamOneP4File(self.stream_file, self.stream_contents)
2814
2815     def make_email(self, userid):
2816         if userid in self.users:
2817             return self.users[userid]
2818         else:
2819             return "%s <a@b>" % userid
2820
2821     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2822         """ Stream a p4 tag.
2823         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2824         """
2825
2826         if verbose:
2827             print "writing tag %s for commit %s" % (labelName, commit)
2828         gitStream.write("tag %s\n" % labelName)
2829         gitStream.write("from %s\n" % commit)
2830
2831         if labelDetails.has_key('Owner'):
2832             owner = labelDetails["Owner"]
2833         else:
2834             owner = None
2835
2836         # Try to use the owner of the p4 label, or failing that,
2837         # the current p4 user id.
2838         if owner:
2839             email = self.make_email(owner)
2840         else:
2841             email = self.make_email(self.p4UserId())
2842         tagger = "%s %s %s" % (email, epoch, self.tz)
2843
2844         gitStream.write("tagger %s\n" % tagger)
2845
2846         print "labelDetails=",labelDetails
2847         if labelDetails.has_key('Description'):
2848             description = labelDetails['Description']
2849         else:
2850             description = 'Label from git p4'
2851
2852         gitStream.write("data %d\n" % len(description))
2853         gitStream.write(description)
2854         gitStream.write("\n")
2855
2856     def inClientSpec(self, path):
2857         if not self.clientSpecDirs:
2858             return True
2859         inClientSpec = self.clientSpecDirs.map_in_client(path)
2860         if not inClientSpec and self.verbose:
2861             print('Ignoring file outside of client spec: {0}'.format(path))
2862         return inClientSpec
2863
2864     def hasBranchPrefix(self, path):
2865         if not self.branchPrefixes:
2866             return True
2867         hasPrefix = [p for p in self.branchPrefixes
2868                         if p4PathStartsWith(path, p)]
2869         if not hasPrefix and self.verbose:
2870             print('Ignoring file outside of prefix: {0}'.format(path))
2871         return hasPrefix
2872
2873     def commit(self, details, files, branch, parent = ""):
2874         epoch = details["time"]
2875         author = details["user"]
2876         jobs = self.extractJobsFromCommit(details)
2877
2878         if self.verbose:
2879             print('commit into {0}'.format(branch))
2880
2881         if self.clientSpecDirs:
2882             self.clientSpecDirs.update_client_spec_path_cache(files)
2883
2884         files = [f for f in files
2885             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2886
2887         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2888             print('Ignoring revision {0} as it would produce an empty commit.'
2889                 .format(details['change']))
2890             return
2891
2892         self.gitStream.write("commit %s\n" % branch)
2893         self.gitStream.write("mark :%s\n" % details["change"])
2894         self.committedChanges.add(int(details["change"]))
2895         committer = ""
2896         if author not in self.users:
2897             self.getUserMapFromPerforceServer()
2898         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2899
2900         self.gitStream.write("committer %s\n" % committer)
2901
2902         self.gitStream.write("data <<EOT\n")
2903         self.gitStream.write(details["desc"])
2904         if len(jobs) > 0:
2905             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2906
2907         if not self.suppress_meta_comment:
2908             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2909                                 (','.join(self.branchPrefixes), details["change"]))
2910             if len(details['options']) > 0:
2911                 self.gitStream.write(": options = %s" % details['options'])
2912             self.gitStream.write("]\n")
2913
2914         self.gitStream.write("EOT\n\n")
2915
2916         if len(parent) > 0:
2917             if self.verbose:
2918                 print "parent %s" % parent
2919             self.gitStream.write("from %s\n" % parent)
2920
2921         self.streamP4Files(files)
2922         self.gitStream.write("\n")
2923
2924         change = int(details["change"])
2925
2926         if self.labels.has_key(change):
2927             label = self.labels[change]
2928             labelDetails = label[0]
2929             labelRevisions = label[1]
2930             if self.verbose:
2931                 print "Change %s is labelled %s" % (change, labelDetails)
2932
2933             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2934                                                 for p in self.branchPrefixes])
2935
2936             if len(files) == len(labelRevisions):
2937
2938                 cleanedFiles = {}
2939                 for info in files:
2940                     if info["action"] in self.delete_actions:
2941                         continue
2942                     cleanedFiles[info["depotFile"]] = info["rev"]
2943
2944                 if cleanedFiles == labelRevisions:
2945                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2946
2947                 else:
2948                     if not self.silent:
2949                         print ("Tag %s does not match with change %s: files do not match."
2950                                % (labelDetails["label"], change))
2951
2952             else:
2953                 if not self.silent:
2954                     print ("Tag %s does not match with change %s: file count is different."
2955                            % (labelDetails["label"], change))
2956
2957     # Build a dictionary of changelists and labels, for "detect-labels" option.
2958     def getLabels(self):
2959         self.labels = {}
2960
2961         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2962         if len(l) > 0 and not self.silent:
2963             print "Finding files belonging to labels in %s" % `self.depotPaths`
2964
2965         for output in l:
2966             label = output["label"]
2967             revisions = {}
2968             newestChange = 0
2969             if self.verbose:
2970                 print "Querying files for label %s" % label
2971             for file in p4CmdList(["files"] +
2972                                       ["%s...@%s" % (p, label)
2973                                           for p in self.depotPaths]):
2974                 revisions[file["depotFile"]] = file["rev"]
2975                 change = int(file["change"])
2976                 if change > newestChange:
2977                     newestChange = change
2978
2979             self.labels[newestChange] = [output, revisions]
2980
2981         if self.verbose:
2982             print "Label changes: %s" % self.labels.keys()
2983
2984     # Import p4 labels as git tags. A direct mapping does not
2985     # exist, so assume that if all the files are at the same revision
2986     # then we can use that, or it's something more complicated we should
2987     # just ignore.
2988     def importP4Labels(self, stream, p4Labels):
2989         if verbose:
2990             print "import p4 labels: " + ' '.join(p4Labels)
2991
2992         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2993         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2994         if len(validLabelRegexp) == 0:
2995             validLabelRegexp = defaultLabelRegexp
2996         m = re.compile(validLabelRegexp)
2997
2998         for name in p4Labels:
2999             commitFound = False
3000
3001             if not m.match(name):
3002                 if verbose:
3003                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
3004                 continue
3005
3006             if name in ignoredP4Labels:
3007                 continue
3008
3009             labelDetails = p4CmdList(['label', "-o", name])[0]
3010
3011             # get the most recent changelist for each file in this label
3012             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3013                                 for p in self.depotPaths])
3014
3015             if change.has_key('change'):
3016                 # find the corresponding git commit; take the oldest commit
3017                 changelist = int(change['change'])
3018                 if changelist in self.committedChanges:
3019                     gitCommit = ":%d" % changelist       # use a fast-import mark
3020                     commitFound = True
3021                 else:
3022                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3023                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3024                     if len(gitCommit) == 0:
3025                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3026                     else:
3027                         commitFound = True
3028                         gitCommit = gitCommit.strip()
3029
3030                 if commitFound:
3031                     # Convert from p4 time format
3032                     try:
3033                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3034                     except ValueError:
3035                         print "Could not convert label time %s" % labelDetails['Update']
3036                         tmwhen = 1
3037
3038                     when = int(time.mktime(tmwhen))
3039                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3040                     if verbose:
3041                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3042             else:
3043                 if verbose:
3044                     print "Label %s has no changelists - possibly deleted?" % name
3045
3046             if not commitFound:
3047                 # We can't import this label; don't try again as it will get very
3048                 # expensive repeatedly fetching all the files for labels that will
3049                 # never be imported. If the label is moved in the future, the
3050                 # ignore will need to be removed manually.
3051                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3052
3053     def guessProjectName(self):
3054         for p in self.depotPaths:
3055             if p.endswith("/"):
3056                 p = p[:-1]
3057             p = p[p.strip().rfind("/") + 1:]
3058             if not p.endswith("/"):
3059                p += "/"
3060             return p
3061
3062     def getBranchMapping(self):
3063         lostAndFoundBranches = set()
3064
3065         user = gitConfig("git-p4.branchUser")
3066         if len(user) > 0:
3067             command = "branches -u %s" % user
3068         else:
3069             command = "branches"
3070
3071         for info in p4CmdList(command):
3072             details = p4Cmd(["branch", "-o", info["branch"]])
3073             viewIdx = 0
3074             while details.has_key("View%s" % viewIdx):
3075                 paths = details["View%s" % viewIdx].split(" ")
3076                 viewIdx = viewIdx + 1
3077                 # require standard //depot/foo/... //depot/bar/... mapping
3078                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3079                     continue
3080                 source = paths[0]
3081                 destination = paths[1]
3082                 ## HACK
3083                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3084                     source = source[len(self.depotPaths[0]):-4]
3085                     destination = destination[len(self.depotPaths[0]):-4]
3086
3087                     if destination in self.knownBranches:
3088                         if not self.silent:
3089                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3090                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3091                         continue
3092
3093                     self.knownBranches[destination] = source
3094
3095                     lostAndFoundBranches.discard(destination)
3096
3097                     if source not in self.knownBranches:
3098                         lostAndFoundBranches.add(source)
3099
3100         # Perforce does not strictly require branches to be defined, so we also
3101         # check git config for a branch list.
3102         #
3103         # Example of branch definition in git config file:
3104         # [git-p4]
3105         #   branchList=main:branchA
3106         #   branchList=main:branchB
3107         #   branchList=branchA:branchC
3108         configBranches = gitConfigList("git-p4.branchList")
3109         for branch in configBranches:
3110             if branch:
3111                 (source, destination) = branch.split(":")
3112                 self.knownBranches[destination] = source
3113
3114                 lostAndFoundBranches.discard(destination)
3115
3116                 if source not in self.knownBranches:
3117                     lostAndFoundBranches.add(source)
3118
3119
3120         for branch in lostAndFoundBranches:
3121             self.knownBranches[branch] = branch
3122
3123     def getBranchMappingFromGitBranches(self):
3124         branches = p4BranchesInGit(self.importIntoRemotes)
3125         for branch in branches.keys():
3126             if branch == "master":
3127                 branch = "main"
3128             else:
3129                 branch = branch[len(self.projectName):]
3130             self.knownBranches[branch] = branch
3131
3132     def updateOptionDict(self, d):
3133         option_keys = {}
3134         if self.keepRepoPath:
3135             option_keys['keepRepoPath'] = 1
3136
3137         d["options"] = ' '.join(sorted(option_keys.keys()))
3138
3139     def readOptions(self, d):
3140         self.keepRepoPath = (d.has_key('options')
3141                              and ('keepRepoPath' in d['options']))
3142
3143     def gitRefForBranch(self, branch):
3144         if branch == "main":
3145             return self.refPrefix + "master"
3146
3147         if len(branch) <= 0:
3148             return branch
3149
3150         return self.refPrefix + self.projectName + branch
3151
3152     def gitCommitByP4Change(self, ref, change):
3153         if self.verbose:
3154             print "looking in ref " + ref + " for change %s using bisect..." % change
3155
3156         earliestCommit = ""
3157         latestCommit = parseRevision(ref)
3158
3159         while True:
3160             if self.verbose:
3161                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3162             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3163             if len(next) == 0:
3164                 if self.verbose:
3165                     print "argh"
3166                 return ""
3167             log = extractLogMessageFromGitCommit(next)
3168             settings = extractSettingsGitLog(log)
3169             currentChange = int(settings['change'])
3170             if self.verbose:
3171                 print "current change %s" % currentChange
3172
3173             if currentChange == change:
3174                 if self.verbose:
3175                     print "found %s" % next
3176                 return next
3177
3178             if currentChange < change:
3179                 earliestCommit = "^%s" % next
3180             else:
3181                 latestCommit = "%s" % next
3182
3183         return ""
3184
3185     def importNewBranch(self, branch, maxChange):
3186         # make fast-import flush all changes to disk and update the refs using the checkpoint
3187         # command so that we can try to find the branch parent in the git history
3188         self.gitStream.write("checkpoint\n\n");
3189         self.gitStream.flush();
3190         branchPrefix = self.depotPaths[0] + branch + "/"
3191         range = "@1,%s" % maxChange
3192         #print "prefix" + branchPrefix
3193         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3194         if len(changes) <= 0:
3195             return False
3196         firstChange = changes[0]
3197         #print "first change in branch: %s" % firstChange
3198         sourceBranch = self.knownBranches[branch]
3199         sourceDepotPath = self.depotPaths[0] + sourceBranch
3200         sourceRef = self.gitRefForBranch(sourceBranch)
3201         #print "source " + sourceBranch
3202
3203         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3204         #print "branch parent: %s" % branchParentChange
3205         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3206         if len(gitParent) > 0:
3207             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3208             #print "parent git commit: %s" % gitParent
3209
3210         self.importChanges(changes)
3211         return True
3212
3213     def searchParent(self, parent, branch, target):
3214         parentFound = False
3215         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3216                                      "--no-merges", parent]):
3217             blob = blob.strip()
3218             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3219                 parentFound = True
3220                 if self.verbose:
3221                     print "Found parent of %s in commit %s" % (branch, blob)
3222                 break
3223         if parentFound:
3224             return blob
3225         else:
3226             return None
3227
3228     def importChanges(self, changes, shelved=False, origin_revision=0):
3229         cnt = 1
3230         for change in changes:
3231             description = p4_describe(change, shelved)
3232             self.updateOptionDict(description)
3233
3234             if not self.silent:
3235                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3236                 sys.stdout.flush()
3237             cnt = cnt + 1
3238
3239             try:
3240                 if self.detectBranches:
3241                     branches = self.splitFilesIntoBranches(description)
3242                     for branch in branches.keys():
3243                         ## HACK  --hwn
3244                         branchPrefix = self.depotPaths[0] + branch + "/"
3245                         self.branchPrefixes = [ branchPrefix ]
3246
3247                         parent = ""
3248
3249                         filesForCommit = branches[branch]
3250
3251                         if self.verbose:
3252                             print "branch is %s" % branch
3253
3254                         self.updatedBranches.add(branch)
3255
3256                         if branch not in self.createdBranches:
3257                             self.createdBranches.add(branch)
3258                             parent = self.knownBranches[branch]
3259                             if parent == branch:
3260                                 parent = ""
3261                             else:
3262                                 fullBranch = self.projectName + branch
3263                                 if fullBranch not in self.p4BranchesInGit:
3264                                     if not self.silent:
3265                                         print("\n    Importing new branch %s" % fullBranch);
3266                                     if self.importNewBranch(branch, change - 1):
3267                                         parent = ""
3268                                         self.p4BranchesInGit.append(fullBranch)
3269                                     if not self.silent:
3270                                         print("\n    Resuming with change %s" % change);
3271
3272                                 if self.verbose:
3273                                     print "parent determined through known branches: %s" % parent
3274
3275                         branch = self.gitRefForBranch(branch)
3276                         parent = self.gitRefForBranch(parent)
3277
3278                         if self.verbose:
3279                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3280
3281                         if len(parent) == 0 and branch in self.initialParents:
3282                             parent = self.initialParents[branch]
3283                             del self.initialParents[branch]
3284
3285                         blob = None
3286                         if len(parent) > 0:
3287                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3288                             if self.verbose:
3289                                 print "Creating temporary branch: " + tempBranch
3290                             self.commit(description, filesForCommit, tempBranch)
3291                             self.tempBranches.append(tempBranch)
3292                             self.checkpoint()
3293                             blob = self.searchParent(parent, branch, tempBranch)
3294                         if blob:
3295                             self.commit(description, filesForCommit, branch, blob)
3296                         else:
3297                             if self.verbose:
3298                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3299                             self.commit(description, filesForCommit, branch, parent)
3300                 else:
3301                     files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
3302                     self.commit(description, files, self.branch,
3303                                 self.initialParent)
3304                     # only needed once, to connect to the previous commit
3305                     self.initialParent = ""
3306             except IOError:
3307                 print self.gitError.read()
3308                 sys.exit(1)
3309
3310     def importHeadRevision(self, revision):
3311         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3312
3313         details = {}
3314         details["user"] = "git perforce import user"
3315         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3316                            % (' '.join(self.depotPaths), revision))
3317         details["change"] = revision
3318         newestRevision = 0
3319
3320         fileCnt = 0
3321         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3322
3323         for info in p4CmdList(["files"] + fileArgs):
3324
3325             if 'code' in info and info['code'] == 'error':
3326                 sys.stderr.write("p4 returned an error: %s\n"
3327                                  % info['data'])
3328                 if info['data'].find("must refer to client") >= 0:
3329                     sys.stderr.write("This particular p4 error is misleading.\n")
3330                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3331                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3332                 sys.exit(1)
3333             if 'p4ExitCode' in info:
3334                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3335                 sys.exit(1)
3336
3337
3338             change = int(info["change"])
3339             if change > newestRevision:
3340                 newestRevision = change
3341
3342             if info["action"] in self.delete_actions:
3343                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3344                 #fileCnt = fileCnt + 1
3345                 continue
3346
3347             for prop in ["depotFile", "rev", "action", "type" ]:
3348                 details["%s%s" % (prop, fileCnt)] = info[prop]
3349
3350             fileCnt = fileCnt + 1
3351
3352         details["change"] = newestRevision
3353
3354         # Use time from top-most change so that all git p4 clones of
3355         # the same p4 repo have the same commit SHA1s.
3356         res = p4_describe(newestRevision)
3357         details["time"] = res["time"]
3358
3359         self.updateOptionDict(details)
3360         try:
3361             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3362         except IOError:
3363             print "IO error with git fast-import. Is your git version recent enough?"
3364             print self.gitError.read()
3365
3366     def openStreams(self):
3367         self.importProcess = subprocess.Popen(["git", "fast-import"],
3368                                               stdin=subprocess.PIPE,
3369                                               stdout=subprocess.PIPE,
3370                                               stderr=subprocess.PIPE);
3371         self.gitOutput = self.importProcess.stdout
3372         self.gitStream = self.importProcess.stdin
3373         self.gitError = self.importProcess.stderr
3374
3375     def closeStreams(self):
3376         self.gitStream.close()
3377         if self.importProcess.wait() != 0:
3378             die("fast-import failed: %s" % self.gitError.read())
3379         self.gitOutput.close()
3380         self.gitError.close()
3381
3382     def run(self, args):
3383         if self.importIntoRemotes:
3384             self.refPrefix = "refs/remotes/p4/"
3385         else:
3386             self.refPrefix = "refs/heads/p4/"
3387
3388         if self.syncWithOrigin:
3389             self.hasOrigin = originP4BranchesExist()
3390             if self.hasOrigin:
3391                 if not self.silent:
3392                     print 'Syncing with origin first, using "git fetch origin"'
3393                 system("git fetch origin")
3394
3395         branch_arg_given = bool(self.branch)
3396         if len(self.branch) == 0:
3397             self.branch = self.refPrefix + "master"
3398             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3399                 system("git update-ref %s refs/heads/p4" % self.branch)
3400                 system("git branch -D p4")
3401
3402         # accept either the command-line option, or the configuration variable
3403         if self.useClientSpec:
3404             # will use this after clone to set the variable
3405             self.useClientSpec_from_options = True
3406         else:
3407             if gitConfigBool("git-p4.useclientspec"):
3408                 self.useClientSpec = True
3409         if self.useClientSpec:
3410             self.clientSpecDirs = getClientSpec()
3411
3412         # TODO: should always look at previous commits,
3413         # merge with previous imports, if possible.
3414         if args == []:
3415             if self.hasOrigin:
3416                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3417
3418             # branches holds mapping from branch name to sha1
3419             branches = p4BranchesInGit(self.importIntoRemotes)
3420
3421             # restrict to just this one, disabling detect-branches
3422             if branch_arg_given:
3423                 short = self.branch.split("/")[-1]
3424                 if short in branches:
3425                     self.p4BranchesInGit = [ short ]
3426             else:
3427                 self.p4BranchesInGit = branches.keys()
3428
3429             if len(self.p4BranchesInGit) > 1:
3430                 if not self.silent:
3431                     print "Importing from/into multiple branches"
3432                 self.detectBranches = True
3433                 for branch in branches.keys():
3434                     self.initialParents[self.refPrefix + branch] = \
3435                         branches[branch]
3436
3437             if self.verbose:
3438                 print "branches: %s" % self.p4BranchesInGit
3439
3440             p4Change = 0
3441             for branch in self.p4BranchesInGit:
3442                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3443
3444                 settings = extractSettingsGitLog(logMsg)
3445
3446                 self.readOptions(settings)
3447                 if (settings.has_key('depot-paths')
3448                     and settings.has_key ('change')):
3449                     change = int(settings['change']) + 1
3450                     p4Change = max(p4Change, change)
3451
3452                     depotPaths = sorted(settings['depot-paths'])
3453                     if self.previousDepotPaths == []:
3454                         self.previousDepotPaths = depotPaths
3455                     else:
3456                         paths = []
3457                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3458                             prev_list = prev.split("/")
3459                             cur_list = cur.split("/")
3460                             for i in range(0, min(len(cur_list), len(prev_list))):
3461                                 if cur_list[i] <> prev_list[i]:
3462                                     i = i - 1
3463                                     break
3464
3465                             paths.append ("/".join(cur_list[:i + 1]))
3466
3467                         self.previousDepotPaths = paths
3468
3469             if p4Change > 0:
3470                 self.depotPaths = sorted(self.previousDepotPaths)
3471                 self.changeRange = "@%s,#head" % p4Change
3472                 if not self.silent and not self.detectBranches:
3473                     print "Performing incremental import into %s git branch" % self.branch
3474
3475         # accept multiple ref name abbreviations:
3476         #    refs/foo/bar/branch -> use it exactly
3477         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3478         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3479         if not self.branch.startswith("refs/"):
3480             if self.importIntoRemotes:
3481                 prepend = "refs/remotes/"
3482             else:
3483                 prepend = "refs/heads/"
3484             if not self.branch.startswith("p4/"):
3485                 prepend += "p4/"
3486             self.branch = prepend + self.branch
3487
3488         if len(args) == 0 and self.depotPaths:
3489             if not self.silent:
3490                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3491         else:
3492             if self.depotPaths and self.depotPaths != args:
3493                 print ("previous import used depot path %s and now %s was specified. "
3494                        "This doesn't work!" % (' '.join (self.depotPaths),
3495                                                ' '.join (args)))
3496                 sys.exit(1)
3497
3498             self.depotPaths = sorted(args)
3499
3500         revision = ""
3501         self.users = {}
3502
3503         # Make sure no revision specifiers are used when --changesfile
3504         # is specified.
3505         bad_changesfile = False
3506         if len(self.changesFile) > 0:
3507             for p in self.depotPaths:
3508                 if p.find("@") >= 0 or p.find("#") >= 0:
3509                     bad_changesfile = True
3510                     break
3511         if bad_changesfile:
3512             die("Option --changesfile is incompatible with revision specifiers")
3513
3514         newPaths = []
3515         for p in self.depotPaths:
3516             if p.find("@") != -1:
3517                 atIdx = p.index("@")
3518                 self.changeRange = p[atIdx:]
3519                 if self.changeRange == "@all":
3520                     self.changeRange = ""
3521                 elif ',' not in self.changeRange:
3522                     revision = self.changeRange
3523                     self.changeRange = ""
3524                 p = p[:atIdx]
3525             elif p.find("#") != -1:
3526                 hashIdx = p.index("#")
3527                 revision = p[hashIdx:]
3528                 p = p[:hashIdx]
3529             elif self.previousDepotPaths == []:
3530                 # pay attention to changesfile, if given, else import
3531                 # the entire p4 tree at the head revision
3532                 if len(self.changesFile) == 0:
3533                     revision = "#head"
3534
3535             p = re.sub ("\.\.\.$", "", p)
3536             if not p.endswith("/"):
3537                 p += "/"
3538
3539             newPaths.append(p)
3540
3541         self.depotPaths = newPaths
3542
3543         # --detect-branches may change this for each branch
3544         self.branchPrefixes = self.depotPaths
3545
3546         self.loadUserMapFromCache()
3547         self.labels = {}
3548         if self.detectLabels:
3549             self.getLabels();
3550
3551         if self.detectBranches:
3552             ## FIXME - what's a P4 projectName ?
3553             self.projectName = self.guessProjectName()
3554
3555             if self.hasOrigin:
3556                 self.getBranchMappingFromGitBranches()
3557             else:
3558                 self.getBranchMapping()
3559             if self.verbose:
3560                 print "p4-git branches: %s" % self.p4BranchesInGit
3561                 print "initial parents: %s" % self.initialParents
3562             for b in self.p4BranchesInGit:
3563                 if b != "master":
3564
3565                     ## FIXME
3566                     b = b[len(self.projectName):]
3567                 self.createdBranches.add(b)
3568
3569         self.openStreams()
3570
3571         if revision:
3572             self.importHeadRevision(revision)
3573         else:
3574             changes = []
3575
3576             if len(self.changesFile) > 0:
3577                 output = open(self.changesFile).readlines()
3578                 changeSet = set()
3579                 for line in output:
3580                     changeSet.add(int(line))
3581
3582                 for change in changeSet:
3583                     changes.append(change)
3584
3585                 changes.sort()
3586             else:
3587                 # catch "git p4 sync" with no new branches, in a repo that
3588                 # does not have any existing p4 branches
3589                 if len(args) == 0:
3590                     if not self.p4BranchesInGit:
3591                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3592
3593                     # The default branch is master, unless --branch is used to
3594                     # specify something else.  Make sure it exists, or complain
3595                     # nicely about how to use --branch.
3596                     if not self.detectBranches:
3597                         if not branch_exists(self.branch):
3598                             if branch_arg_given:
3599                                 die("Error: branch %s does not exist." % self.branch)
3600                             else:
3601                                 die("Error: no branch %s; perhaps specify one with --branch." %
3602                                     self.branch)
3603
3604                 if self.verbose:
3605                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3606                                                               self.changeRange)
3607                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3608
3609                 if len(self.maxChanges) > 0:
3610                     changes = changes[:min(int(self.maxChanges), len(changes))]
3611
3612             if len(changes) == 0:
3613                 if not self.silent:
3614                     print "No changes to import!"
3615             else:
3616                 if not self.silent and not self.detectBranches:
3617                     print "Import destination: %s" % self.branch
3618
3619                 self.updatedBranches = set()
3620
3621                 if not self.detectBranches:
3622                     if args:
3623                         # start a new branch
3624                         self.initialParent = ""
3625                     else:
3626                         # build on a previous revision
3627                         self.initialParent = parseRevision(self.branch)
3628
3629                 self.importChanges(changes)
3630
3631                 if not self.silent:
3632                     print ""
3633                     if len(self.updatedBranches) > 0:
3634                         sys.stdout.write("Updated branches: ")
3635                         for b in self.updatedBranches:
3636                             sys.stdout.write("%s " % b)
3637                         sys.stdout.write("\n")
3638
3639         if gitConfigBool("git-p4.importLabels"):
3640             self.importLabels = True
3641
3642         if self.importLabels:
3643             p4Labels = getP4Labels(self.depotPaths)
3644             gitTags = getGitTags()
3645
3646             missingP4Labels = p4Labels - gitTags
3647             self.importP4Labels(self.gitStream, missingP4Labels)
3648
3649         self.closeStreams()
3650
3651         # Cleanup temporary branches created during import
3652         if self.tempBranches != []:
3653             for branch in self.tempBranches:
3654                 read_pipe("git update-ref -d %s" % branch)
3655             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3656
3657         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3658         # a convenient shortcut refname "p4".
3659         if self.importIntoRemotes:
3660             head_ref = self.refPrefix + "HEAD"
3661             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3662                 system(["git", "symbolic-ref", head_ref, self.branch])
3663
3664         return True
3665
3666 class P4Rebase(Command):
3667     def __init__(self):
3668         Command.__init__(self)
3669         self.options = [
3670                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3671         ]
3672         self.importLabels = False
3673         self.description = ("Fetches the latest revision from perforce and "
3674                             + "rebases the current work (branch) against it")
3675
3676     def run(self, args):
3677         sync = P4Sync()
3678         sync.importLabels = self.importLabels
3679         sync.run([])
3680
3681         return self.rebase()
3682
3683     def rebase(self):
3684         if os.system("git update-index --refresh") != 0:
3685             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3686         if len(read_pipe("git diff-index HEAD --")) > 0:
3687             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3688
3689         [upstream, settings] = findUpstreamBranchPoint()
3690         if len(upstream) == 0:
3691             die("Cannot find upstream branchpoint for rebase")
3692
3693         # the branchpoint may be p4/foo~3, so strip off the parent
3694         upstream = re.sub("~[0-9]+$", "", upstream)
3695
3696         print "Rebasing the current branch onto %s" % upstream
3697         oldHead = read_pipe("git rev-parse HEAD").strip()
3698         system("git rebase %s" % upstream)
3699         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3700         return True
3701
3702 class P4Clone(P4Sync):
3703     def __init__(self):
3704         P4Sync.__init__(self)
3705         self.description = "Creates a new git repository and imports from Perforce into it"
3706         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3707         self.options += [
3708             optparse.make_option("--destination", dest="cloneDestination",
3709                                  action='store', default=None,
3710                                  help="where to leave result of the clone"),
3711             optparse.make_option("--bare", dest="cloneBare",
3712                                  action="store_true", default=False),
3713         ]
3714         self.cloneDestination = None
3715         self.needsGit = False
3716         self.cloneBare = False
3717
3718     def defaultDestination(self, args):
3719         ## TODO: use common prefix of args?
3720         depotPath = args[0]
3721         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3722         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3723         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3724         depotDir = re.sub(r"/$", "", depotDir)
3725         return os.path.split(depotDir)[1]
3726
3727     def run(self, args):
3728         if len(args) < 1:
3729             return False
3730
3731         if self.keepRepoPath and not self.cloneDestination:
3732             sys.stderr.write("Must specify destination for --keep-path\n")
3733             sys.exit(1)
3734
3735         depotPaths = args
3736
3737         if not self.cloneDestination and len(depotPaths) > 1:
3738             self.cloneDestination = depotPaths[-1]
3739             depotPaths = depotPaths[:-1]
3740
3741         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3742         for p in depotPaths:
3743             if not p.startswith("//"):
3744                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3745                 return False
3746
3747         if not self.cloneDestination:
3748             self.cloneDestination = self.defaultDestination(args)
3749
3750         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3751
3752         if not os.path.exists(self.cloneDestination):
3753             os.makedirs(self.cloneDestination)
3754         chdir(self.cloneDestination)
3755
3756         init_cmd = [ "git", "init" ]
3757         if self.cloneBare:
3758             init_cmd.append("--bare")
3759         retcode = subprocess.call(init_cmd)
3760         if retcode:
3761             raise CalledProcessError(retcode, init_cmd)
3762
3763         if not P4Sync.run(self, depotPaths):
3764             return False
3765
3766         # create a master branch and check out a work tree
3767         if gitBranchExists(self.branch):
3768             system([ "git", "branch", "master", self.branch ])
3769             if not self.cloneBare:
3770                 system([ "git", "checkout", "-f" ])
3771         else:
3772             print 'Not checking out any branch, use ' \
3773                   '"git checkout -q -b master <branch>"'
3774
3775         # auto-set this variable if invoked with --use-client-spec
3776         if self.useClientSpec_from_options:
3777             system("git config --bool git-p4.useclientspec true")
3778
3779         return True
3780
3781 class P4Unshelve(Command):
3782     def __init__(self):
3783         Command.__init__(self)
3784         self.options = []
3785         self.origin = "HEAD"
3786         self.description = "Unshelve a P4 changelist into a git commit"
3787         self.usage = "usage: %prog [options] changelist"
3788         self.options += [
3789                 optparse.make_option("--origin", dest="origin",
3790                     help="Use this base revision instead of the default (%s)" % self.origin),
3791         ]
3792         self.verbose = False
3793         self.noCommit = False
3794         self.destbranch = "refs/remotes/p4/unshelved"
3795
3796     def renameBranch(self, branch_name):
3797         """ Rename the existing branch to branch_name.N
3798         """
3799
3800         found = True
3801         for i in range(0,1000):
3802             backup_branch_name = "{0}.{1}".format(branch_name, i)
3803             if not gitBranchExists(backup_branch_name):
3804                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3805                 gitDeleteRef(branch_name)
3806                 found = True
3807                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3808                 break
3809
3810         if not found:
3811             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3812
3813     def findLastP4Revision(self, starting_point):
3814         """ Look back from starting_point for the first commit created by git-p4
3815             to find the P4 commit we are based on, and the depot-paths.
3816         """
3817
3818         for parent in (range(65535)):
3819             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3820             settings = extractSettingsGitLog(log)
3821             if settings.has_key('change'):
3822                 return settings
3823
3824         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3825
3826     def run(self, args):
3827         if len(args) != 1:
3828             return False
3829
3830         if not gitBranchExists(self.origin):
3831             sys.exit("origin branch {0} does not exist".format(self.origin))
3832
3833         sync = P4Sync()
3834         changes = args
3835         sync.initialParent = self.origin
3836
3837         # use the first change in the list to construct the branch to unshelve into
3838         change = changes[0]
3839
3840         # if the target branch already exists, rename it
3841         branch_name = "{0}/{1}".format(self.destbranch, change)
3842         if gitBranchExists(branch_name):
3843             self.renameBranch(branch_name)
3844         sync.branch = branch_name
3845
3846         sync.verbose = self.verbose
3847         sync.suppress_meta_comment = True
3848
3849         settings = self.findLastP4Revision(self.origin)
3850         origin_revision = settings['change']
3851         sync.depotPaths = settings['depot-paths']
3852         sync.branchPrefixes = sync.depotPaths
3853
3854         sync.openStreams()
3855         sync.loadUserMapFromCache()
3856         sync.silent = True
3857         sync.importChanges(changes, shelved=True, origin_revision=origin_revision)
3858         sync.closeStreams()
3859
3860         print("unshelved changelist {0} into {1}".format(change, branch_name))
3861
3862         return True
3863
3864 class P4Branches(Command):
3865     def __init__(self):
3866         Command.__init__(self)
3867         self.options = [ ]
3868         self.description = ("Shows the git branches that hold imports and their "
3869                             + "corresponding perforce depot paths")
3870         self.verbose = False
3871
3872     def run(self, args):
3873         if originP4BranchesExist():
3874             createOrUpdateBranchesFromOrigin()
3875
3876         cmdline = "git rev-parse --symbolic "
3877         cmdline += " --remotes"
3878
3879         for line in read_pipe_lines(cmdline):
3880             line = line.strip()
3881
3882             if not line.startswith('p4/') or line == "p4/HEAD":
3883                 continue
3884             branch = line
3885
3886             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3887             settings = extractSettingsGitLog(log)
3888
3889             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3890         return True
3891
3892 class HelpFormatter(optparse.IndentedHelpFormatter):
3893     def __init__(self):
3894         optparse.IndentedHelpFormatter.__init__(self)
3895
3896     def format_description(self, description):
3897         if description:
3898             return description + "\n"
3899         else:
3900             return ""
3901
3902 def printUsage(commands):
3903     print "usage: %s <command> [options]" % sys.argv[0]
3904     print ""
3905     print "valid commands: %s" % ", ".join(commands)
3906     print ""
3907     print "Try %s <command> --help for command specific help." % sys.argv[0]
3908     print ""
3909
3910 commands = {
3911     "debug" : P4Debug,
3912     "submit" : P4Submit,
3913     "commit" : P4Submit,
3914     "sync" : P4Sync,
3915     "rebase" : P4Rebase,
3916     "clone" : P4Clone,
3917     "rollback" : P4RollBack,
3918     "branches" : P4Branches,
3919     "unshelve" : P4Unshelve,
3920 }
3921
3922
3923 def main():
3924     if len(sys.argv[1:]) == 0:
3925         printUsage(commands.keys())
3926         sys.exit(2)
3927
3928     cmdName = sys.argv[1]
3929     try:
3930         klass = commands[cmdName]
3931         cmd = klass()
3932     except KeyError:
3933         print "unknown command %s" % cmdName
3934         print ""
3935         printUsage(commands.keys())
3936         sys.exit(2)
3937
3938     options = cmd.options
3939     cmd.gitdir = os.environ.get("GIT_DIR", None)
3940
3941     args = sys.argv[2:]
3942
3943     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3944     if cmd.needsGit:
3945         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3946
3947     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3948                                    options,
3949                                    description = cmd.description,
3950                                    formatter = HelpFormatter())
3951
3952     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3953     global verbose
3954     verbose = cmd.verbose
3955     if cmd.needsGit:
3956         if cmd.gitdir == None:
3957             cmd.gitdir = os.path.abspath(".git")
3958             if not isValidGitDir(cmd.gitdir):
3959                 # "rev-parse --git-dir" without arguments will try $PWD/.git
3960                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3961                 if os.path.exists(cmd.gitdir):
3962                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3963                     if len(cdup) > 0:
3964                         chdir(cdup);
3965
3966         if not isValidGitDir(cmd.gitdir):
3967             if isValidGitDir(cmd.gitdir + "/.git"):
3968                 cmd.gitdir += "/.git"
3969             else:
3970                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3971
3972         # so git commands invoked from the P4 workspace will succeed
3973         os.environ["GIT_DIR"] = cmd.gitdir
3974
3975     if not cmd.run(args):
3976         parser.print_help()
3977         sys.exit(2)
3978
3979
3980 if __name__ == '__main__':
3981     main()