pack-objects: move in_pack_pos out of struct object_entry
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31     from subprocess import CalledProcessError
32 except ImportError:
33     # from python2.7:subprocess.py
34     # Exception classes used by this module.
35     class CalledProcessError(Exception):
36         """This exception is raised when a process run by check_call() returns
37         a non-zero exit status.  The exit status will be stored in the
38         returncode attribute."""
39         def __init__(self, returncode, cmd):
40             self.returncode = returncode
41             self.cmd = cmd
42         def __str__(self):
43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54     """Build a suitable p4 command line.
55
56     This consolidates building and returning a p4 command line into one
57     location. It means that hooking into the environment, or other configuration
58     can be done more easily.
59     """
60     real_cmd = ["p4"]
61
62     user = gitConfig("git-p4.user")
63     if len(user) > 0:
64         real_cmd += ["-u",user]
65
66     password = gitConfig("git-p4.password")
67     if len(password) > 0:
68         real_cmd += ["-P", password]
69
70     port = gitConfig("git-p4.port")
71     if len(port) > 0:
72         real_cmd += ["-p", port]
73
74     host = gitConfig("git-p4.host")
75     if len(host) > 0:
76         real_cmd += ["-H", host]
77
78     client = gitConfig("git-p4.client")
79     if len(client) > 0:
80         real_cmd += ["-c", client]
81
82     retries = gitConfigInt("git-p4.retries")
83     if retries is None:
84         # Perform 3 retries by default
85         retries = 3
86     if retries > 0:
87         # Provide a way to not pass this option by setting git-p4.retries to 0
88         real_cmd += ["-r", str(retries)]
89
90     if isinstance(cmd,basestring):
91         real_cmd = ' '.join(real_cmd) + ' ' + cmd
92     else:
93         real_cmd += cmd
94     return real_cmd
95
96 def git_dir(path):
97     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98         This won't automatically add ".git" to a directory.
99     """
100     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101     if not d or len(d) == 0:
102         return None
103     else:
104         return d
105
106 def chdir(path, is_client_path=False):
107     """Do chdir to the given path, and set the PWD environment
108        variable for use by P4.  It does not look at getcwd() output.
109        Since we're not using the shell, it is necessary to set the
110        PWD environment variable explicitly.
111
112        Normally, expand the path to force it to be absolute.  This
113        addresses the use of relative path names inside P4 settings,
114        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
115        as given; it looks for .p4config using PWD.
116
117        If is_client_path, the path was handed to us directly by p4,
118        and may be a symbolic link.  Do not call os.getcwd() in this
119        case, because it will cause p4 to think that PWD is not inside
120        the client path.
121        """
122
123     os.chdir(path)
124     if not is_client_path:
125         path = os.getcwd()
126     os.environ['PWD'] = path
127
128 def calcDiskFree():
129     """Return free space in bytes on the disk of the given dirname."""
130     if platform.system() == 'Windows':
131         free_bytes = ctypes.c_ulonglong(0)
132         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133         return free_bytes.value
134     else:
135         st = os.statvfs(os.getcwd())
136         return st.f_bavail * st.f_frsize
137
138 def die(msg):
139     if verbose:
140         raise Exception(msg)
141     else:
142         sys.stderr.write(msg + "\n")
143         sys.exit(1)
144
145 def write_pipe(c, stdin):
146     if verbose:
147         sys.stderr.write('Writing pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151     pipe = p.stdin
152     val = pipe.write(stdin)
153     pipe.close()
154     if p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_write_pipe(c, stdin):
160     real_cmd = p4_build_cmd(c)
161     return write_pipe(real_cmd, stdin)
162
163 def read_pipe_full(c):
164     """ Read output from  command. Returns a tuple
165         of the return status, stdout text and stderr
166         text.
167     """
168     if verbose:
169         sys.stderr.write('Reading pipe: %s\n' % str(c))
170
171     expand = isinstance(c,basestring)
172     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173     (out, err) = p.communicate()
174     return (p.returncode, out, err)
175
176 def read_pipe(c, ignore_error=False):
177     """ Read output from  command. Returns the output text on
178         success. On failure, terminates execution, unless
179         ignore_error is True, when it returns an empty string.
180     """
181     (retcode, out, err) = read_pipe_full(c)
182     if retcode != 0:
183         if ignore_error:
184             out = ""
185         else:
186             die('Command failed: %s\nError: %s' % (str(c), err))
187     return out
188
189 def read_pipe_text(c):
190     """ Read output from a command with trailing whitespace stripped.
191         On error, returns None.
192     """
193     (retcode, out, err) = read_pipe_full(c)
194     if retcode != 0:
195         return None
196     else:
197         return out.rstrip()
198
199 def p4_read_pipe(c, ignore_error=False):
200     real_cmd = p4_build_cmd(c)
201     return read_pipe(real_cmd, ignore_error)
202
203 def read_pipe_lines(c):
204     if verbose:
205         sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207     expand = isinstance(c, basestring)
208     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209     pipe = p.stdout
210     val = pipe.readlines()
211     if pipe.close() or p.wait():
212         die('Command failed: %s' % str(c))
213
214     return val
215
216 def p4_read_pipe_lines(c):
217     """Specifically invoke p4 on the command supplied. """
218     real_cmd = p4_build_cmd(c)
219     return read_pipe_lines(real_cmd)
220
221 def p4_has_command(cmd):
222     """Ask p4 for help on this command.  If it returns an error, the
223        command does not exist in this version of p4."""
224     real_cmd = p4_build_cmd(["help", cmd])
225     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226                                    stderr=subprocess.PIPE)
227     p.communicate()
228     return p.returncode == 0
229
230 def p4_has_move_command():
231     """See if the move command exists, that it supports -k, and that
232        it has not been administratively disabled.  The arguments
233        must be correct, but the filenames do not have to exist.  Use
234        ones with wildcards so even if they exist, it will fail."""
235
236     if not p4_has_command("move"):
237         return False
238     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240     (out, err) = p.communicate()
241     # return code will be 1 in either case
242     if err.find("Invalid option") >= 0:
243         return False
244     if err.find("disabled") >= 0:
245         return False
246     # assume it failed because @... was invalid changelist
247     return True
248
249 def system(cmd, ignore_error=False):
250     expand = isinstance(cmd,basestring)
251     if verbose:
252         sys.stderr.write("executing %s\n" % str(cmd))
253     retcode = subprocess.call(cmd, shell=expand)
254     if retcode and not ignore_error:
255         raise CalledProcessError(retcode, cmd)
256
257     return retcode
258
259 def p4_system(cmd):
260     """Specifically invoke p4 as the system command. """
261     real_cmd = p4_build_cmd(cmd)
262     expand = isinstance(real_cmd, basestring)
263     retcode = subprocess.call(real_cmd, shell=expand)
264     if retcode:
265         raise CalledProcessError(retcode, real_cmd)
266
267 _p4_version_string = None
268 def p4_version_string():
269     """Read the version string, showing just the last line, which
270        hopefully is the interesting version bit.
271
272        $ p4 -V
273        Perforce - The Fast Software Configuration Management System.
274        Copyright 1995-2011 Perforce Software.  All rights reserved.
275        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276     """
277     global _p4_version_string
278     if not _p4_version_string:
279         a = p4_read_pipe_lines(["-V"])
280         _p4_version_string = a[-1].rstrip()
281     return _p4_version_string
282
283 def p4_integrate(src, dest):
284     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
285
286 def p4_sync(f, *options):
287     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
288
289 def p4_add(f):
290     # forcibly add file names with wildcards
291     if wildcard_present(f):
292         p4_system(["add", "-f", f])
293     else:
294         p4_system(["add", f])
295
296 def p4_delete(f):
297     p4_system(["delete", wildcard_encode(f)])
298
299 def p4_edit(f, *options):
300     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
301
302 def p4_revert(f):
303     p4_system(["revert", wildcard_encode(f)])
304
305 def p4_reopen(type, f):
306     p4_system(["reopen", "-t", type, wildcard_encode(f)])
307
308 def p4_reopen_in_change(changelist, files):
309     cmd = ["reopen", "-c", str(changelist)] + files
310     p4_system(cmd)
311
312 def p4_move(src, dest):
313     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
315 def p4_last_change():
316     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317     return int(results[0]['change'])
318
319 def p4_describe(change):
320     """Make sure it returns a valid result by checking for
321        the presence of field "time".  Return a dict of the
322        results."""
323
324     ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
325     if len(ds) != 1:
326         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328     d = ds[0]
329
330     if "p4ExitCode" in d:
331         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332                                                       str(d)))
333     if "code" in d:
334         if d["code"] == "error":
335             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337     if "time" not in d:
338         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340     return d
341
342 #
343 # Canonicalize the p4 type and return a tuple of the
344 # base type, plus any modifiers.  See "p4 help filetypes"
345 # for a list and explanation.
346 #
347 def split_p4_type(p4type):
348
349     p4_filetypes_historical = {
350         "ctempobj": "binary+Sw",
351         "ctext": "text+C",
352         "cxtext": "text+Cx",
353         "ktext": "text+k",
354         "kxtext": "text+kx",
355         "ltext": "text+F",
356         "tempobj": "binary+FSw",
357         "ubinary": "binary+F",
358         "uresource": "resource+F",
359         "uxbinary": "binary+Fx",
360         "xbinary": "binary+x",
361         "xltext": "text+Fx",
362         "xtempobj": "binary+Swx",
363         "xtext": "text+x",
364         "xunicode": "unicode+x",
365         "xutf16": "utf16+x",
366     }
367     if p4type in p4_filetypes_historical:
368         p4type = p4_filetypes_historical[p4type]
369     mods = ""
370     s = p4type.split("+")
371     base = s[0]
372     mods = ""
373     if len(s) > 1:
374         mods = s[1]
375     return (base, mods)
376
377 #
378 # return the raw p4 type of a file (text, text+ko, etc)
379 #
380 def p4_type(f):
381     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
382     return results[0]['headType']
383
384 #
385 # Given a type base and modifier, return a regexp matching
386 # the keywords that can be expanded in the file
387 #
388 def p4_keywords_regexp_for_type(base, type_mods):
389     if base in ("text", "unicode", "binary"):
390         kwords = None
391         if "ko" in type_mods:
392             kwords = 'Id|Header'
393         elif "k" in type_mods:
394             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395         else:
396             return None
397         pattern = r"""
398             \$              # Starts with a dollar, followed by...
399             (%s)            # one of the keywords, followed by...
400             (:[^$\n]+)?     # possibly an old expansion, followed by...
401             \$              # another dollar
402             """ % kwords
403         return pattern
404     else:
405         return None
406
407 #
408 # Given a file, return a regexp matching the possible
409 # RCS keywords that will be expanded, or None for files
410 # with kw expansion turned off.
411 #
412 def p4_keywords_regexp_for_file(file):
413     if not os.path.exists(file):
414         return None
415     else:
416         (type_base, type_mods) = split_p4_type(p4_type(file))
417         return p4_keywords_regexp_for_type(type_base, type_mods)
418
419 def setP4ExecBit(file, mode):
420     # Reopens an already open file and changes the execute bit to match
421     # the execute bit setting in the passed in mode.
422
423     p4Type = "+x"
424
425     if not isModeExec(mode):
426         p4Type = getP4OpenedType(file)
427         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429         if p4Type[-1] == "+":
430             p4Type = p4Type[0:-1]
431
432     p4_reopen(p4Type, file)
433
434 def getP4OpenedType(file):
435     # Returns the perforce file type for the given file.
436
437     result = p4_read_pipe(["opened", wildcard_encode(file)])
438     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
439     if match:
440         return match.group(1)
441     else:
442         die("Could not determine file type for %s (result: '%s')" % (file, result))
443
444 # Return the set of all p4 labels
445 def getP4Labels(depotPaths):
446     labels = set()
447     if isinstance(depotPaths,basestring):
448         depotPaths = [depotPaths]
449
450     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451         label = l['label']
452         labels.add(label)
453
454     return labels
455
456 # Return the set of all git tags
457 def getGitTags():
458     gitTags = set()
459     for line in read_pipe_lines(["git", "tag"]):
460         tag = line.strip()
461         gitTags.add(tag)
462     return gitTags
463
464 def diffTreePattern():
465     # This is a simple generator for the diff tree regex pattern. This could be
466     # a class variable if this and parseDiffTreeEntry were a part of a class.
467     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468     while True:
469         yield pattern
470
471 def parseDiffTreeEntry(entry):
472     """Parses a single diff tree entry into its component elements.
473
474     See git-diff-tree(1) manpage for details about the format of the diff
475     output. This method returns a dictionary with the following elements:
476
477     src_mode - The mode of the source file
478     dst_mode - The mode of the destination file
479     src_sha1 - The sha1 for the source file
480     dst_sha1 - The sha1 fr the destination file
481     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482     status_score - The score for the status (applicable for 'C' and 'R'
483                    statuses). This is None if there is no score.
484     src - The path for the source file.
485     dst - The path for the destination file. This is only present for
486           copy or renames. If it is not present, this is None.
487
488     If the pattern is not matched, None is returned."""
489
490     match = diffTreePattern().next().match(entry)
491     if match:
492         return {
493             'src_mode': match.group(1),
494             'dst_mode': match.group(2),
495             'src_sha1': match.group(3),
496             'dst_sha1': match.group(4),
497             'status': match.group(5),
498             'status_score': match.group(6),
499             'src': match.group(7),
500             'dst': match.group(10)
501         }
502     return None
503
504 def isModeExec(mode):
505     # Returns True if the given git mode represents an executable file,
506     # otherwise False.
507     return mode[-3:] == "755"
508
509 def isModeExecChanged(src_mode, dst_mode):
510     return isModeExec(src_mode) != isModeExec(dst_mode)
511
512 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
513
514     if isinstance(cmd,basestring):
515         cmd = "-G " + cmd
516         expand = True
517     else:
518         cmd = ["-G"] + cmd
519         expand = False
520
521     cmd = p4_build_cmd(cmd)
522     if verbose:
523         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
524
525     # Use a temporary file to avoid deadlocks without
526     # subprocess.communicate(), which would put another copy
527     # of stdout into memory.
528     stdin_file = None
529     if stdin is not None:
530         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
531         if isinstance(stdin,basestring):
532             stdin_file.write(stdin)
533         else:
534             for i in stdin:
535                 stdin_file.write(i + '\n')
536         stdin_file.flush()
537         stdin_file.seek(0)
538
539     p4 = subprocess.Popen(cmd,
540                           shell=expand,
541                           stdin=stdin_file,
542                           stdout=subprocess.PIPE)
543
544     result = []
545     try:
546         while True:
547             entry = marshal.load(p4.stdout)
548             if skip_info:
549                 if 'code' in entry and entry['code'] == 'info':
550                     continue
551             if cb is not None:
552                 cb(entry)
553             else:
554                 result.append(entry)
555     except EOFError:
556         pass
557     exitCode = p4.wait()
558     if exitCode != 0:
559         entry = {}
560         entry["p4ExitCode"] = exitCode
561         result.append(entry)
562
563     return result
564
565 def p4Cmd(cmd):
566     list = p4CmdList(cmd)
567     result = {}
568     for entry in list:
569         result.update(entry)
570     return result;
571
572 def p4Where(depotPath):
573     if not depotPath.endswith("/"):
574         depotPath += "/"
575     depotPathLong = depotPath + "..."
576     outputList = p4CmdList(["where", depotPathLong])
577     output = None
578     for entry in outputList:
579         if "depotFile" in entry:
580             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
581             # The base path always ends with "/...".
582             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
583                 output = entry
584                 break
585         elif "data" in entry:
586             data = entry.get("data")
587             space = data.find(" ")
588             if data[:space] == depotPath:
589                 output = entry
590                 break
591     if output == None:
592         return ""
593     if output["code"] == "error":
594         return ""
595     clientPath = ""
596     if "path" in output:
597         clientPath = output.get("path")
598     elif "data" in output:
599         data = output.get("data")
600         lastSpace = data.rfind(" ")
601         clientPath = data[lastSpace + 1:]
602
603     if clientPath.endswith("..."):
604         clientPath = clientPath[:-3]
605     return clientPath
606
607 def currentGitBranch():
608     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
609
610 def isValidGitDir(path):
611     return git_dir(path) != None
612
613 def parseRevision(ref):
614     return read_pipe("git rev-parse %s" % ref).strip()
615
616 def branchExists(ref):
617     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
618                      ignore_error=True)
619     return len(rev) > 0
620
621 def extractLogMessageFromGitCommit(commit):
622     logMessage = ""
623
624     ## fixme: title is first line of commit, not 1st paragraph.
625     foundTitle = False
626     for log in read_pipe_lines("git cat-file commit %s" % commit):
627        if not foundTitle:
628            if len(log) == 1:
629                foundTitle = True
630            continue
631
632        logMessage += log
633     return logMessage
634
635 def extractSettingsGitLog(log):
636     values = {}
637     for line in log.split("\n"):
638         line = line.strip()
639         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
640         if not m:
641             continue
642
643         assignments = m.group(1).split (':')
644         for a in assignments:
645             vals = a.split ('=')
646             key = vals[0].strip()
647             val = ('='.join (vals[1:])).strip()
648             if val.endswith ('\"') and val.startswith('"'):
649                 val = val[1:-1]
650
651             values[key] = val
652
653     paths = values.get("depot-paths")
654     if not paths:
655         paths = values.get("depot-path")
656     if paths:
657         values['depot-paths'] = paths.split(',')
658     return values
659
660 def gitBranchExists(branch):
661     proc = subprocess.Popen(["git", "rev-parse", branch],
662                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
663     return proc.wait() == 0;
664
665 _gitConfig = {}
666
667 def gitConfig(key, typeSpecifier=None):
668     if not _gitConfig.has_key(key):
669         cmd = [ "git", "config" ]
670         if typeSpecifier:
671             cmd += [ typeSpecifier ]
672         cmd += [ key ]
673         s = read_pipe(cmd, ignore_error=True)
674         _gitConfig[key] = s.strip()
675     return _gitConfig[key]
676
677 def gitConfigBool(key):
678     """Return a bool, using git config --bool.  It is True only if the
679        variable is set to true, and False if set to false or not present
680        in the config."""
681
682     if not _gitConfig.has_key(key):
683         _gitConfig[key] = gitConfig(key, '--bool') == "true"
684     return _gitConfig[key]
685
686 def gitConfigInt(key):
687     if not _gitConfig.has_key(key):
688         cmd = [ "git", "config", "--int", key ]
689         s = read_pipe(cmd, ignore_error=True)
690         v = s.strip()
691         try:
692             _gitConfig[key] = int(gitConfig(key, '--int'))
693         except ValueError:
694             _gitConfig[key] = None
695     return _gitConfig[key]
696
697 def gitConfigList(key):
698     if not _gitConfig.has_key(key):
699         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
700         _gitConfig[key] = s.strip().splitlines()
701         if _gitConfig[key] == ['']:
702             _gitConfig[key] = []
703     return _gitConfig[key]
704
705 def p4BranchesInGit(branchesAreInRemotes=True):
706     """Find all the branches whose names start with "p4/", looking
707        in remotes or heads as specified by the argument.  Return
708        a dictionary of { branch: revision } for each one found.
709        The branch names are the short names, without any
710        "p4/" prefix."""
711
712     branches = {}
713
714     cmdline = "git rev-parse --symbolic "
715     if branchesAreInRemotes:
716         cmdline += "--remotes"
717     else:
718         cmdline += "--branches"
719
720     for line in read_pipe_lines(cmdline):
721         line = line.strip()
722
723         # only import to p4/
724         if not line.startswith('p4/'):
725             continue
726         # special symbolic ref to p4/master
727         if line == "p4/HEAD":
728             continue
729
730         # strip off p4/ prefix
731         branch = line[len("p4/"):]
732
733         branches[branch] = parseRevision(line)
734
735     return branches
736
737 def branch_exists(branch):
738     """Make sure that the given ref name really exists."""
739
740     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
741     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
742     out, _ = p.communicate()
743     if p.returncode:
744         return False
745     # expect exactly one line of output: the branch name
746     return out.rstrip() == branch
747
748 def findUpstreamBranchPoint(head = "HEAD"):
749     branches = p4BranchesInGit()
750     # map from depot-path to branch name
751     branchByDepotPath = {}
752     for branch in branches.keys():
753         tip = branches[branch]
754         log = extractLogMessageFromGitCommit(tip)
755         settings = extractSettingsGitLog(log)
756         if settings.has_key("depot-paths"):
757             paths = ",".join(settings["depot-paths"])
758             branchByDepotPath[paths] = "remotes/p4/" + branch
759
760     settings = None
761     parent = 0
762     while parent < 65535:
763         commit = head + "~%s" % parent
764         log = extractLogMessageFromGitCommit(commit)
765         settings = extractSettingsGitLog(log)
766         if settings.has_key("depot-paths"):
767             paths = ",".join(settings["depot-paths"])
768             if branchByDepotPath.has_key(paths):
769                 return [branchByDepotPath[paths], settings]
770
771         parent = parent + 1
772
773     return ["", settings]
774
775 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
776     if not silent:
777         print ("Creating/updating branch(es) in %s based on origin branch(es)"
778                % localRefPrefix)
779
780     originPrefix = "origin/p4/"
781
782     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
783         line = line.strip()
784         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
785             continue
786
787         headName = line[len(originPrefix):]
788         remoteHead = localRefPrefix + headName
789         originHead = line
790
791         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
792         if (not original.has_key('depot-paths')
793             or not original.has_key('change')):
794             continue
795
796         update = False
797         if not gitBranchExists(remoteHead):
798             if verbose:
799                 print "creating %s" % remoteHead
800             update = True
801         else:
802             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
803             if settings.has_key('change') > 0:
804                 if settings['depot-paths'] == original['depot-paths']:
805                     originP4Change = int(original['change'])
806                     p4Change = int(settings['change'])
807                     if originP4Change > p4Change:
808                         print ("%s (%s) is newer than %s (%s). "
809                                "Updating p4 branch from origin."
810                                % (originHead, originP4Change,
811                                   remoteHead, p4Change))
812                         update = True
813                 else:
814                     print ("Ignoring: %s was imported from %s while "
815                            "%s was imported from %s"
816                            % (originHead, ','.join(original['depot-paths']),
817                               remoteHead, ','.join(settings['depot-paths'])))
818
819         if update:
820             system("git update-ref %s %s" % (remoteHead, originHead))
821
822 def originP4BranchesExist():
823         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
824
825
826 def p4ParseNumericChangeRange(parts):
827     changeStart = int(parts[0][1:])
828     if parts[1] == '#head':
829         changeEnd = p4_last_change()
830     else:
831         changeEnd = int(parts[1])
832
833     return (changeStart, changeEnd)
834
835 def chooseBlockSize(blockSize):
836     if blockSize:
837         return blockSize
838     else:
839         return defaultBlockSize
840
841 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
842     assert depotPaths
843
844     # Parse the change range into start and end. Try to find integer
845     # revision ranges as these can be broken up into blocks to avoid
846     # hitting server-side limits (maxrows, maxscanresults). But if
847     # that doesn't work, fall back to using the raw revision specifier
848     # strings, without using block mode.
849
850     if changeRange is None or changeRange == '':
851         changeStart = 1
852         changeEnd = p4_last_change()
853         block_size = chooseBlockSize(requestedBlockSize)
854     else:
855         parts = changeRange.split(',')
856         assert len(parts) == 2
857         try:
858             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
859             block_size = chooseBlockSize(requestedBlockSize)
860         except:
861             changeStart = parts[0][1:]
862             changeEnd = parts[1]
863             if requestedBlockSize:
864                 die("cannot use --changes-block-size with non-numeric revisions")
865             block_size = None
866
867     changes = set()
868
869     # Retrieve changes a block at a time, to prevent running
870     # into a MaxResults/MaxScanRows error from the server.
871
872     while True:
873         cmd = ['changes']
874
875         if block_size:
876             end = min(changeEnd, changeStart + block_size)
877             revisionRange = "%d,%d" % (changeStart, end)
878         else:
879             revisionRange = "%s,%s" % (changeStart, changeEnd)
880
881         for p in depotPaths:
882             cmd += ["%s...@%s" % (p, revisionRange)]
883
884         # Insert changes in chronological order
885         for entry in reversed(p4CmdList(cmd)):
886             if entry.has_key('p4ExitCode'):
887                 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
888             if not entry.has_key('change'):
889                 continue
890             changes.add(int(entry['change']))
891
892         if not block_size:
893             break
894
895         if end >= changeEnd:
896             break
897
898         changeStart = end + 1
899
900     changes = sorted(changes)
901     return changes
902
903 def p4PathStartsWith(path, prefix):
904     # This method tries to remedy a potential mixed-case issue:
905     #
906     # If UserA adds  //depot/DirA/file1
907     # and UserB adds //depot/dira/file2
908     #
909     # we may or may not have a problem. If you have core.ignorecase=true,
910     # we treat DirA and dira as the same directory
911     if gitConfigBool("core.ignorecase"):
912         return path.lower().startswith(prefix.lower())
913     return path.startswith(prefix)
914
915 def getClientSpec():
916     """Look at the p4 client spec, create a View() object that contains
917        all the mappings, and return it."""
918
919     specList = p4CmdList("client -o")
920     if len(specList) != 1:
921         die('Output from "client -o" is %d lines, expecting 1' %
922             len(specList))
923
924     # dictionary of all client parameters
925     entry = specList[0]
926
927     # the //client/ name
928     client_name = entry["Client"]
929
930     # just the keys that start with "View"
931     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
932
933     # hold this new View
934     view = View(client_name)
935
936     # append the lines, in order, to the view
937     for view_num in range(len(view_keys)):
938         k = "View%d" % view_num
939         if k not in view_keys:
940             die("Expected view key %s missing" % k)
941         view.append(entry[k])
942
943     return view
944
945 def getClientRoot():
946     """Grab the client directory."""
947
948     output = p4CmdList("client -o")
949     if len(output) != 1:
950         die('Output from "client -o" is %d lines, expecting 1' % len(output))
951
952     entry = output[0]
953     if "Root" not in entry:
954         die('Client has no "Root"')
955
956     return entry["Root"]
957
958 #
959 # P4 wildcards are not allowed in filenames.  P4 complains
960 # if you simply add them, but you can force it with "-f", in
961 # which case it translates them into %xx encoding internally.
962 #
963 def wildcard_decode(path):
964     # Search for and fix just these four characters.  Do % last so
965     # that fixing it does not inadvertently create new %-escapes.
966     # Cannot have * in a filename in windows; untested as to
967     # what p4 would do in such a case.
968     if not platform.system() == "Windows":
969         path = path.replace("%2A", "*")
970     path = path.replace("%23", "#") \
971                .replace("%40", "@") \
972                .replace("%25", "%")
973     return path
974
975 def wildcard_encode(path):
976     # do % first to avoid double-encoding the %s introduced here
977     path = path.replace("%", "%25") \
978                .replace("*", "%2A") \
979                .replace("#", "%23") \
980                .replace("@", "%40")
981     return path
982
983 def wildcard_present(path):
984     m = re.search("[*#@%]", path)
985     return m is not None
986
987 class LargeFileSystem(object):
988     """Base class for large file system support."""
989
990     def __init__(self, writeToGitStream):
991         self.largeFiles = set()
992         self.writeToGitStream = writeToGitStream
993
994     def generatePointer(self, cloneDestination, contentFile):
995         """Return the content of a pointer file that is stored in Git instead of
996            the actual content."""
997         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
998
999     def pushFile(self, localLargeFile):
1000         """Push the actual content which is not stored in the Git repository to
1001            a server."""
1002         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1003
1004     def hasLargeFileExtension(self, relPath):
1005         return reduce(
1006             lambda a, b: a or b,
1007             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1008             False
1009         )
1010
1011     def generateTempFile(self, contents):
1012         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1013         for d in contents:
1014             contentFile.write(d)
1015         contentFile.close()
1016         return contentFile.name
1017
1018     def exceedsLargeFileThreshold(self, relPath, contents):
1019         if gitConfigInt('git-p4.largeFileThreshold'):
1020             contentsSize = sum(len(d) for d in contents)
1021             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1022                 return True
1023         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024             contentsSize = sum(len(d) for d in contents)
1025             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1026                 return False
1027             contentTempFile = self.generateTempFile(contents)
1028             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1029             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1030             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1031             zf.close()
1032             compressedContentsSize = zf.infolist()[0].compress_size
1033             os.remove(contentTempFile)
1034             os.remove(compressedContentFile.name)
1035             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1036                 return True
1037         return False
1038
1039     def addLargeFile(self, relPath):
1040         self.largeFiles.add(relPath)
1041
1042     def removeLargeFile(self, relPath):
1043         self.largeFiles.remove(relPath)
1044
1045     def isLargeFile(self, relPath):
1046         return relPath in self.largeFiles
1047
1048     def processContent(self, git_mode, relPath, contents):
1049         """Processes the content of git fast import. This method decides if a
1050            file is stored in the large file system and handles all necessary
1051            steps."""
1052         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1053             contentTempFile = self.generateTempFile(contents)
1054             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1055             if pointer_git_mode:
1056                 git_mode = pointer_git_mode
1057             if localLargeFile:
1058                 # Move temp file to final location in large file system
1059                 largeFileDir = os.path.dirname(localLargeFile)
1060                 if not os.path.isdir(largeFileDir):
1061                     os.makedirs(largeFileDir)
1062                 shutil.move(contentTempFile, localLargeFile)
1063                 self.addLargeFile(relPath)
1064                 if gitConfigBool('git-p4.largeFilePush'):
1065                     self.pushFile(localLargeFile)
1066                 if verbose:
1067                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1068         return (git_mode, contents)
1069
1070 class MockLFS(LargeFileSystem):
1071     """Mock large file system for testing."""
1072
1073     def generatePointer(self, contentFile):
1074         """The pointer content is the original content prefixed with "pointer-".
1075            The local filename of the large file storage is derived from the file content.
1076            """
1077         with open(contentFile, 'r') as f:
1078             content = next(f)
1079             gitMode = '100644'
1080             pointerContents = 'pointer-' + content
1081             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1082             return (gitMode, pointerContents, localLargeFile)
1083
1084     def pushFile(self, localLargeFile):
1085         """The remote filename of the large file storage is the same as the local
1086            one but in a different directory.
1087            """
1088         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1089         if not os.path.exists(remotePath):
1090             os.makedirs(remotePath)
1091         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1092
1093 class GitLFS(LargeFileSystem):
1094     """Git LFS as backend for the git-p4 large file system.
1095        See https://git-lfs.github.com/ for details."""
1096
1097     def __init__(self, *args):
1098         LargeFileSystem.__init__(self, *args)
1099         self.baseGitAttributes = []
1100
1101     def generatePointer(self, contentFile):
1102         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1103            mode and content which is stored in the Git repository instead of
1104            the actual content. Return also the new location of the actual
1105            content.
1106            """
1107         if os.path.getsize(contentFile) == 0:
1108             return (None, '', None)
1109
1110         pointerProcess = subprocess.Popen(
1111             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1112             stdout=subprocess.PIPE
1113         )
1114         pointerFile = pointerProcess.stdout.read()
1115         if pointerProcess.wait():
1116             os.remove(contentFile)
1117             die('git-lfs pointer command failed. Did you install the extension?')
1118
1119         # Git LFS removed the preamble in the output of the 'pointer' command
1120         # starting from version 1.2.0. Check for the preamble here to support
1121         # earlier versions.
1122         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1123         if pointerFile.startswith('Git LFS pointer for'):
1124             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1125
1126         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1127         localLargeFile = os.path.join(
1128             os.getcwd(),
1129             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1130             oid,
1131         )
1132         # LFS Spec states that pointer files should not have the executable bit set.
1133         gitMode = '100644'
1134         return (gitMode, pointerFile, localLargeFile)
1135
1136     def pushFile(self, localLargeFile):
1137         uploadProcess = subprocess.Popen(
1138             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1139         )
1140         if uploadProcess.wait():
1141             die('git-lfs push command failed. Did you define a remote?')
1142
1143     def generateGitAttributes(self):
1144         return (
1145             self.baseGitAttributes +
1146             [
1147                 '\n',
1148                 '#\n',
1149                 '# Git LFS (see https://git-lfs.github.com/)\n',
1150                 '#\n',
1151             ] +
1152             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1153                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1154             ] +
1155             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1156                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1157             ]
1158         )
1159
1160     def addLargeFile(self, relPath):
1161         LargeFileSystem.addLargeFile(self, relPath)
1162         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1163
1164     def removeLargeFile(self, relPath):
1165         LargeFileSystem.removeLargeFile(self, relPath)
1166         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1167
1168     def processContent(self, git_mode, relPath, contents):
1169         if relPath == '.gitattributes':
1170             self.baseGitAttributes = contents
1171             return (git_mode, self.generateGitAttributes())
1172         else:
1173             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1174
1175 class Command:
1176     def __init__(self):
1177         self.usage = "usage: %prog [options]"
1178         self.needsGit = True
1179         self.verbose = False
1180
1181     # This is required for the "append" cloneExclude action
1182     def ensure_value(self, attr, value):
1183         if not hasattr(self, attr) or getattr(self, attr) is None:
1184             setattr(self, attr, value)
1185         return getattr(self, attr)
1186
1187 class P4UserMap:
1188     def __init__(self):
1189         self.userMapFromPerforceServer = False
1190         self.myP4UserId = None
1191
1192     def p4UserId(self):
1193         if self.myP4UserId:
1194             return self.myP4UserId
1195
1196         results = p4CmdList("user -o")
1197         for r in results:
1198             if r.has_key('User'):
1199                 self.myP4UserId = r['User']
1200                 return r['User']
1201         die("Could not find your p4 user id")
1202
1203     def p4UserIsMe(self, p4User):
1204         # return True if the given p4 user is actually me
1205         me = self.p4UserId()
1206         if not p4User or p4User != me:
1207             return False
1208         else:
1209             return True
1210
1211     def getUserCacheFilename(self):
1212         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1213         return home + "/.gitp4-usercache.txt"
1214
1215     def getUserMapFromPerforceServer(self):
1216         if self.userMapFromPerforceServer:
1217             return
1218         self.users = {}
1219         self.emails = {}
1220
1221         for output in p4CmdList("users"):
1222             if not output.has_key("User"):
1223                 continue
1224             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1225             self.emails[output["Email"]] = output["User"]
1226
1227         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1228         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1229             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1230             if mapUser and len(mapUser[0]) == 3:
1231                 user = mapUser[0][0]
1232                 fullname = mapUser[0][1]
1233                 email = mapUser[0][2]
1234                 self.users[user] = fullname + " <" + email + ">"
1235                 self.emails[email] = user
1236
1237         s = ''
1238         for (key, val) in self.users.items():
1239             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1240
1241         open(self.getUserCacheFilename(), "wb").write(s)
1242         self.userMapFromPerforceServer = True
1243
1244     def loadUserMapFromCache(self):
1245         self.users = {}
1246         self.userMapFromPerforceServer = False
1247         try:
1248             cache = open(self.getUserCacheFilename(), "rb")
1249             lines = cache.readlines()
1250             cache.close()
1251             for line in lines:
1252                 entry = line.strip().split("\t")
1253                 self.users[entry[0]] = entry[1]
1254         except IOError:
1255             self.getUserMapFromPerforceServer()
1256
1257 class P4Debug(Command):
1258     def __init__(self):
1259         Command.__init__(self)
1260         self.options = []
1261         self.description = "A tool to debug the output of p4 -G."
1262         self.needsGit = False
1263
1264     def run(self, args):
1265         j = 0
1266         for output in p4CmdList(args):
1267             print 'Element: %d' % j
1268             j += 1
1269             print output
1270         return True
1271
1272 class P4RollBack(Command):
1273     def __init__(self):
1274         Command.__init__(self)
1275         self.options = [
1276             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1277         ]
1278         self.description = "A tool to debug the multi-branch import. Don't use :)"
1279         self.rollbackLocalBranches = False
1280
1281     def run(self, args):
1282         if len(args) != 1:
1283             return False
1284         maxChange = int(args[0])
1285
1286         if "p4ExitCode" in p4Cmd("changes -m 1"):
1287             die("Problems executing p4");
1288
1289         if self.rollbackLocalBranches:
1290             refPrefix = "refs/heads/"
1291             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1292         else:
1293             refPrefix = "refs/remotes/"
1294             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1295
1296         for line in lines:
1297             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1298                 line = line.strip()
1299                 ref = refPrefix + line
1300                 log = extractLogMessageFromGitCommit(ref)
1301                 settings = extractSettingsGitLog(log)
1302
1303                 depotPaths = settings['depot-paths']
1304                 change = settings['change']
1305
1306                 changed = False
1307
1308                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1309                                                            for p in depotPaths]))) == 0:
1310                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1311                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1312                     continue
1313
1314                 while change and int(change) > maxChange:
1315                     changed = True
1316                     if self.verbose:
1317                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1318                     system("git update-ref %s \"%s^\"" % (ref, ref))
1319                     log = extractLogMessageFromGitCommit(ref)
1320                     settings =  extractSettingsGitLog(log)
1321
1322
1323                     depotPaths = settings['depot-paths']
1324                     change = settings['change']
1325
1326                 if changed:
1327                     print "%s rewound to %s" % (ref, change)
1328
1329         return True
1330
1331 class P4Submit(Command, P4UserMap):
1332
1333     conflict_behavior_choices = ("ask", "skip", "quit")
1334
1335     def __init__(self):
1336         Command.__init__(self)
1337         P4UserMap.__init__(self)
1338         self.options = [
1339                 optparse.make_option("--origin", dest="origin"),
1340                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1341                 # preserve the user, requires relevant p4 permissions
1342                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1343                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1344                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1345                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1346                 optparse.make_option("--conflict", dest="conflict_behavior",
1347                                      choices=self.conflict_behavior_choices),
1348                 optparse.make_option("--branch", dest="branch"),
1349                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1350                                      help="Shelve instead of submit. Shelved files are reverted, "
1351                                      "restoring the workspace to the state before the shelve"),
1352                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1353                                      metavar="CHANGELIST",
1354                                      help="update an existing shelved changelist, implies --shelve, "
1355                                            "repeat in-order for multiple shelved changelists")
1356         ]
1357         self.description = "Submit changes from git to the perforce depot."
1358         self.usage += " [name of git branch to submit into perforce depot]"
1359         self.origin = ""
1360         self.detectRenames = False
1361         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1362         self.dry_run = False
1363         self.shelve = False
1364         self.update_shelve = list()
1365         self.prepare_p4_only = False
1366         self.conflict_behavior = None
1367         self.isWindows = (platform.system() == "Windows")
1368         self.exportLabels = False
1369         self.p4HasMoveCommand = p4_has_move_command()
1370         self.branch = None
1371
1372         if gitConfig('git-p4.largeFileSystem'):
1373             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1374
1375     def check(self):
1376         if len(p4CmdList("opened ...")) > 0:
1377             die("You have files opened with perforce! Close them before starting the sync.")
1378
1379     def separate_jobs_from_description(self, message):
1380         """Extract and return a possible Jobs field in the commit
1381            message.  It goes into a separate section in the p4 change
1382            specification.
1383
1384            A jobs line starts with "Jobs:" and looks like a new field
1385            in a form.  Values are white-space separated on the same
1386            line or on following lines that start with a tab.
1387
1388            This does not parse and extract the full git commit message
1389            like a p4 form.  It just sees the Jobs: line as a marker
1390            to pass everything from then on directly into the p4 form,
1391            but outside the description section.
1392
1393            Return a tuple (stripped log message, jobs string)."""
1394
1395         m = re.search(r'^Jobs:', message, re.MULTILINE)
1396         if m is None:
1397             return (message, None)
1398
1399         jobtext = message[m.start():]
1400         stripped_message = message[:m.start()].rstrip()
1401         return (stripped_message, jobtext)
1402
1403     def prepareLogMessage(self, template, message, jobs):
1404         """Edits the template returned from "p4 change -o" to insert
1405            the message in the Description field, and the jobs text in
1406            the Jobs field."""
1407         result = ""
1408
1409         inDescriptionSection = False
1410
1411         for line in template.split("\n"):
1412             if line.startswith("#"):
1413                 result += line + "\n"
1414                 continue
1415
1416             if inDescriptionSection:
1417                 if line.startswith("Files:") or line.startswith("Jobs:"):
1418                     inDescriptionSection = False
1419                     # insert Jobs section
1420                     if jobs:
1421                         result += jobs + "\n"
1422                 else:
1423                     continue
1424             else:
1425                 if line.startswith("Description:"):
1426                     inDescriptionSection = True
1427                     line += "\n"
1428                     for messageLine in message.split("\n"):
1429                         line += "\t" + messageLine + "\n"
1430
1431             result += line + "\n"
1432
1433         return result
1434
1435     def patchRCSKeywords(self, file, pattern):
1436         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1437         (handle, outFileName) = tempfile.mkstemp(dir='.')
1438         try:
1439             outFile = os.fdopen(handle, "w+")
1440             inFile = open(file, "r")
1441             regexp = re.compile(pattern, re.VERBOSE)
1442             for line in inFile.readlines():
1443                 line = regexp.sub(r'$\1$', line)
1444                 outFile.write(line)
1445             inFile.close()
1446             outFile.close()
1447             # Forcibly overwrite the original file
1448             os.unlink(file)
1449             shutil.move(outFileName, file)
1450         except:
1451             # cleanup our temporary file
1452             os.unlink(outFileName)
1453             print "Failed to strip RCS keywords in %s" % file
1454             raise
1455
1456         print "Patched up RCS keywords in %s" % file
1457
1458     def p4UserForCommit(self,id):
1459         # Return the tuple (perforce user,git email) for a given git commit id
1460         self.getUserMapFromPerforceServer()
1461         gitEmail = read_pipe(["git", "log", "--max-count=1",
1462                               "--format=%ae", id])
1463         gitEmail = gitEmail.strip()
1464         if not self.emails.has_key(gitEmail):
1465             return (None,gitEmail)
1466         else:
1467             return (self.emails[gitEmail],gitEmail)
1468
1469     def checkValidP4Users(self,commits):
1470         # check if any git authors cannot be mapped to p4 users
1471         for id in commits:
1472             (user,email) = self.p4UserForCommit(id)
1473             if not user:
1474                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1475                 if gitConfigBool("git-p4.allowMissingP4Users"):
1476                     print "%s" % msg
1477                 else:
1478                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1479
1480     def lastP4Changelist(self):
1481         # Get back the last changelist number submitted in this client spec. This
1482         # then gets used to patch up the username in the change. If the same
1483         # client spec is being used by multiple processes then this might go
1484         # wrong.
1485         results = p4CmdList("client -o")        # find the current client
1486         client = None
1487         for r in results:
1488             if r.has_key('Client'):
1489                 client = r['Client']
1490                 break
1491         if not client:
1492             die("could not get client spec")
1493         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1494         for r in results:
1495             if r.has_key('change'):
1496                 return r['change']
1497         die("Could not get changelist number for last submit - cannot patch up user details")
1498
1499     def modifyChangelistUser(self, changelist, newUser):
1500         # fixup the user field of a changelist after it has been submitted.
1501         changes = p4CmdList("change -o %s" % changelist)
1502         if len(changes) != 1:
1503             die("Bad output from p4 change modifying %s to user %s" %
1504                 (changelist, newUser))
1505
1506         c = changes[0]
1507         if c['User'] == newUser: return   # nothing to do
1508         c['User'] = newUser
1509         input = marshal.dumps(c)
1510
1511         result = p4CmdList("change -f -i", stdin=input)
1512         for r in result:
1513             if r.has_key('code'):
1514                 if r['code'] == 'error':
1515                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1516             if r.has_key('data'):
1517                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1518                 return
1519         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1520
1521     def canChangeChangelists(self):
1522         # check to see if we have p4 admin or super-user permissions, either of
1523         # which are required to modify changelists.
1524         results = p4CmdList(["protects", self.depotPath])
1525         for r in results:
1526             if r.has_key('perm'):
1527                 if r['perm'] == 'admin':
1528                     return 1
1529                 if r['perm'] == 'super':
1530                     return 1
1531         return 0
1532
1533     def prepareSubmitTemplate(self, changelist=None):
1534         """Run "p4 change -o" to grab a change specification template.
1535            This does not use "p4 -G", as it is nice to keep the submission
1536            template in original order, since a human might edit it.
1537
1538            Remove lines in the Files section that show changes to files
1539            outside the depot path we're committing into."""
1540
1541         [upstream, settings] = findUpstreamBranchPoint()
1542
1543         template = """\
1544 # A Perforce Change Specification.
1545 #
1546 #  Change:      The change number. 'new' on a new changelist.
1547 #  Date:        The date this specification was last modified.
1548 #  Client:      The client on which the changelist was created.  Read-only.
1549 #  User:        The user who created the changelist.
1550 #  Status:      Either 'pending' or 'submitted'. Read-only.
1551 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1552 #  Description: Comments about the changelist.  Required.
1553 #  Jobs:        What opened jobs are to be closed by this changelist.
1554 #               You may delete jobs from this list.  (New changelists only.)
1555 #  Files:       What opened files from the default changelist are to be added
1556 #               to this changelist.  You may delete files from this list.
1557 #               (New changelists only.)
1558 """
1559         files_list = []
1560         inFilesSection = False
1561         change_entry = None
1562         args = ['change', '-o']
1563         if changelist:
1564             args.append(str(changelist))
1565         for entry in p4CmdList(args):
1566             if not entry.has_key('code'):
1567                 continue
1568             if entry['code'] == 'stat':
1569                 change_entry = entry
1570                 break
1571         if not change_entry:
1572             die('Failed to decode output of p4 change -o')
1573         for key, value in change_entry.iteritems():
1574             if key.startswith('File'):
1575                 if settings.has_key('depot-paths'):
1576                     if not [p for p in settings['depot-paths']
1577                             if p4PathStartsWith(value, p)]:
1578                         continue
1579                 else:
1580                     if not p4PathStartsWith(value, self.depotPath):
1581                         continue
1582                 files_list.append(value)
1583                 continue
1584         # Output in the order expected by prepareLogMessage
1585         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1586             if not change_entry.has_key(key):
1587                 continue
1588             template += '\n'
1589             template += key + ':'
1590             if key == 'Description':
1591                 template += '\n'
1592             for field_line in change_entry[key].splitlines():
1593                 template += '\t'+field_line+'\n'
1594         if len(files_list) > 0:
1595             template += '\n'
1596             template += 'Files:\n'
1597         for path in files_list:
1598             template += '\t'+path+'\n'
1599         return template
1600
1601     def edit_template(self, template_file):
1602         """Invoke the editor to let the user change the submission
1603            message.  Return true if okay to continue with the submit."""
1604
1605         # if configured to skip the editing part, just submit
1606         if gitConfigBool("git-p4.skipSubmitEdit"):
1607             return True
1608
1609         # look at the modification time, to check later if the user saved
1610         # the file
1611         mtime = os.stat(template_file).st_mtime
1612
1613         # invoke the editor
1614         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1615             editor = os.environ.get("P4EDITOR")
1616         else:
1617             editor = read_pipe("git var GIT_EDITOR").strip()
1618         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1619
1620         # If the file was not saved, prompt to see if this patch should
1621         # be skipped.  But skip this verification step if configured so.
1622         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1623             return True
1624
1625         # modification time updated means user saved the file
1626         if os.stat(template_file).st_mtime > mtime:
1627             return True
1628
1629         while True:
1630             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1631             if response == 'y':
1632                 return True
1633             if response == 'n':
1634                 return False
1635
1636     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1637         # diff
1638         if os.environ.has_key("P4DIFF"):
1639             del(os.environ["P4DIFF"])
1640         diff = ""
1641         for editedFile in editedFiles:
1642             diff += p4_read_pipe(['diff', '-du',
1643                                   wildcard_encode(editedFile)])
1644
1645         # new file diff
1646         newdiff = ""
1647         for newFile in filesToAdd:
1648             newdiff += "==== new file ====\n"
1649             newdiff += "--- /dev/null\n"
1650             newdiff += "+++ %s\n" % newFile
1651
1652             is_link = os.path.islink(newFile)
1653             expect_link = newFile in symlinks
1654
1655             if is_link and expect_link:
1656                 newdiff += "+%s\n" % os.readlink(newFile)
1657             else:
1658                 f = open(newFile, "r")
1659                 for line in f.readlines():
1660                     newdiff += "+" + line
1661                 f.close()
1662
1663         return (diff + newdiff).replace('\r\n', '\n')
1664
1665     def applyCommit(self, id):
1666         """Apply one commit, return True if it succeeded."""
1667
1668         print "Applying", read_pipe(["git", "show", "-s",
1669                                      "--format=format:%h %s", id])
1670
1671         (p4User, gitEmail) = self.p4UserForCommit(id)
1672
1673         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1674         filesToAdd = set()
1675         filesToChangeType = set()
1676         filesToDelete = set()
1677         editedFiles = set()
1678         pureRenameCopy = set()
1679         symlinks = set()
1680         filesToChangeExecBit = {}
1681         all_files = list()
1682
1683         for line in diff:
1684             diff = parseDiffTreeEntry(line)
1685             modifier = diff['status']
1686             path = diff['src']
1687             all_files.append(path)
1688
1689             if modifier == "M":
1690                 p4_edit(path)
1691                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1692                     filesToChangeExecBit[path] = diff['dst_mode']
1693                 editedFiles.add(path)
1694             elif modifier == "A":
1695                 filesToAdd.add(path)
1696                 filesToChangeExecBit[path] = diff['dst_mode']
1697                 if path in filesToDelete:
1698                     filesToDelete.remove(path)
1699
1700                 dst_mode = int(diff['dst_mode'], 8)
1701                 if dst_mode == 0120000:
1702                     symlinks.add(path)
1703
1704             elif modifier == "D":
1705                 filesToDelete.add(path)
1706                 if path in filesToAdd:
1707                     filesToAdd.remove(path)
1708             elif modifier == "C":
1709                 src, dest = diff['src'], diff['dst']
1710                 p4_integrate(src, dest)
1711                 pureRenameCopy.add(dest)
1712                 if diff['src_sha1'] != diff['dst_sha1']:
1713                     p4_edit(dest)
1714                     pureRenameCopy.discard(dest)
1715                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1716                     p4_edit(dest)
1717                     pureRenameCopy.discard(dest)
1718                     filesToChangeExecBit[dest] = diff['dst_mode']
1719                 if self.isWindows:
1720                     # turn off read-only attribute
1721                     os.chmod(dest, stat.S_IWRITE)
1722                 os.unlink(dest)
1723                 editedFiles.add(dest)
1724             elif modifier == "R":
1725                 src, dest = diff['src'], diff['dst']
1726                 if self.p4HasMoveCommand:
1727                     p4_edit(src)        # src must be open before move
1728                     p4_move(src, dest)  # opens for (move/delete, move/add)
1729                 else:
1730                     p4_integrate(src, dest)
1731                     if diff['src_sha1'] != diff['dst_sha1']:
1732                         p4_edit(dest)
1733                     else:
1734                         pureRenameCopy.add(dest)
1735                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1736                     if not self.p4HasMoveCommand:
1737                         p4_edit(dest)   # with move: already open, writable
1738                     filesToChangeExecBit[dest] = diff['dst_mode']
1739                 if not self.p4HasMoveCommand:
1740                     if self.isWindows:
1741                         os.chmod(dest, stat.S_IWRITE)
1742                     os.unlink(dest)
1743                     filesToDelete.add(src)
1744                 editedFiles.add(dest)
1745             elif modifier == "T":
1746                 filesToChangeType.add(path)
1747             else:
1748                 die("unknown modifier %s for %s" % (modifier, path))
1749
1750         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1751         patchcmd = diffcmd + " | git apply "
1752         tryPatchCmd = patchcmd + "--check -"
1753         applyPatchCmd = patchcmd + "--check --apply -"
1754         patch_succeeded = True
1755
1756         if os.system(tryPatchCmd) != 0:
1757             fixed_rcs_keywords = False
1758             patch_succeeded = False
1759             print "Unfortunately applying the change failed!"
1760
1761             # Patch failed, maybe it's just RCS keyword woes. Look through
1762             # the patch to see if that's possible.
1763             if gitConfigBool("git-p4.attemptRCSCleanup"):
1764                 file = None
1765                 pattern = None
1766                 kwfiles = {}
1767                 for file in editedFiles | filesToDelete:
1768                     # did this file's delta contain RCS keywords?
1769                     pattern = p4_keywords_regexp_for_file(file)
1770
1771                     if pattern:
1772                         # this file is a possibility...look for RCS keywords.
1773                         regexp = re.compile(pattern, re.VERBOSE)
1774                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1775                             if regexp.search(line):
1776                                 if verbose:
1777                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1778                                 kwfiles[file] = pattern
1779                                 break
1780
1781                 for file in kwfiles:
1782                     if verbose:
1783                         print "zapping %s with %s" % (line,pattern)
1784                     # File is being deleted, so not open in p4.  Must
1785                     # disable the read-only bit on windows.
1786                     if self.isWindows and file not in editedFiles:
1787                         os.chmod(file, stat.S_IWRITE)
1788                     self.patchRCSKeywords(file, kwfiles[file])
1789                     fixed_rcs_keywords = True
1790
1791             if fixed_rcs_keywords:
1792                 print "Retrying the patch with RCS keywords cleaned up"
1793                 if os.system(tryPatchCmd) == 0:
1794                     patch_succeeded = True
1795
1796         if not patch_succeeded:
1797             for f in editedFiles:
1798                 p4_revert(f)
1799             return False
1800
1801         #
1802         # Apply the patch for real, and do add/delete/+x handling.
1803         #
1804         system(applyPatchCmd)
1805
1806         for f in filesToChangeType:
1807             p4_edit(f, "-t", "auto")
1808         for f in filesToAdd:
1809             p4_add(f)
1810         for f in filesToDelete:
1811             p4_revert(f)
1812             p4_delete(f)
1813
1814         # Set/clear executable bits
1815         for f in filesToChangeExecBit.keys():
1816             mode = filesToChangeExecBit[f]
1817             setP4ExecBit(f, mode)
1818
1819         update_shelve = 0
1820         if len(self.update_shelve) > 0:
1821             update_shelve = self.update_shelve.pop(0)
1822             p4_reopen_in_change(update_shelve, all_files)
1823
1824         #
1825         # Build p4 change description, starting with the contents
1826         # of the git commit message.
1827         #
1828         logMessage = extractLogMessageFromGitCommit(id)
1829         logMessage = logMessage.strip()
1830         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1831
1832         template = self.prepareSubmitTemplate(update_shelve)
1833         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1834
1835         if self.preserveUser:
1836            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1837
1838         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1839             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1840             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1841             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1842
1843         separatorLine = "######## everything below this line is just the diff #######\n"
1844         if not self.prepare_p4_only:
1845             submitTemplate += separatorLine
1846             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1847
1848         (handle, fileName) = tempfile.mkstemp()
1849         tmpFile = os.fdopen(handle, "w+b")
1850         if self.isWindows:
1851             submitTemplate = submitTemplate.replace("\n", "\r\n")
1852         tmpFile.write(submitTemplate)
1853         tmpFile.close()
1854
1855         if self.prepare_p4_only:
1856             #
1857             # Leave the p4 tree prepared, and the submit template around
1858             # and let the user decide what to do next
1859             #
1860             print
1861             print "P4 workspace prepared for submission."
1862             print "To submit or revert, go to client workspace"
1863             print "  " + self.clientPath
1864             print
1865             print "To submit, use \"p4 submit\" to write a new description,"
1866             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1867                   " \"git p4\"." % fileName
1868             print "You can delete the file \"%s\" when finished." % fileName
1869
1870             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1871                 print "To preserve change ownership by user %s, you must\n" \
1872                       "do \"p4 change -f <change>\" after submitting and\n" \
1873                       "edit the User field."
1874             if pureRenameCopy:
1875                 print "After submitting, renamed files must be re-synced."
1876                 print "Invoke \"p4 sync -f\" on each of these files:"
1877                 for f in pureRenameCopy:
1878                     print "  " + f
1879
1880             print
1881             print "To revert the changes, use \"p4 revert ...\", and delete"
1882             print "the submit template file \"%s\"" % fileName
1883             if filesToAdd:
1884                 print "Since the commit adds new files, they must be deleted:"
1885                 for f in filesToAdd:
1886                     print "  " + f
1887             print
1888             return True
1889
1890         #
1891         # Let the user edit the change description, then submit it.
1892         #
1893         submitted = False
1894
1895         try:
1896             if self.edit_template(fileName):
1897                 # read the edited message and submit
1898                 tmpFile = open(fileName, "rb")
1899                 message = tmpFile.read()
1900                 tmpFile.close()
1901                 if self.isWindows:
1902                     message = message.replace("\r\n", "\n")
1903                 submitTemplate = message[:message.index(separatorLine)]
1904
1905                 if update_shelve:
1906                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1907                 elif self.shelve:
1908                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1909                 else:
1910                     p4_write_pipe(['submit', '-i'], submitTemplate)
1911                     # The rename/copy happened by applying a patch that created a
1912                     # new file.  This leaves it writable, which confuses p4.
1913                     for f in pureRenameCopy:
1914                         p4_sync(f, "-f")
1915
1916                 if self.preserveUser:
1917                     if p4User:
1918                         # Get last changelist number. Cannot easily get it from
1919                         # the submit command output as the output is
1920                         # unmarshalled.
1921                         changelist = self.lastP4Changelist()
1922                         self.modifyChangelistUser(changelist, p4User)
1923
1924                 submitted = True
1925
1926         finally:
1927             # skip this patch
1928             if not submitted or self.shelve:
1929                 if self.shelve:
1930                     print ("Reverting shelved files.")
1931                 else:
1932                     print ("Submission cancelled, undoing p4 changes.")
1933                 for f in editedFiles | filesToDelete:
1934                     p4_revert(f)
1935                 for f in filesToAdd:
1936                     p4_revert(f)
1937                     os.remove(f)
1938
1939         os.remove(fileName)
1940         return submitted
1941
1942     # Export git tags as p4 labels. Create a p4 label and then tag
1943     # with that.
1944     def exportGitTags(self, gitTags):
1945         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1946         if len(validLabelRegexp) == 0:
1947             validLabelRegexp = defaultLabelRegexp
1948         m = re.compile(validLabelRegexp)
1949
1950         for name in gitTags:
1951
1952             if not m.match(name):
1953                 if verbose:
1954                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1955                 continue
1956
1957             # Get the p4 commit this corresponds to
1958             logMessage = extractLogMessageFromGitCommit(name)
1959             values = extractSettingsGitLog(logMessage)
1960
1961             if not values.has_key('change'):
1962                 # a tag pointing to something not sent to p4; ignore
1963                 if verbose:
1964                     print "git tag %s does not give a p4 commit" % name
1965                 continue
1966             else:
1967                 changelist = values['change']
1968
1969             # Get the tag details.
1970             inHeader = True
1971             isAnnotated = False
1972             body = []
1973             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1974                 l = l.strip()
1975                 if inHeader:
1976                     if re.match(r'tag\s+', l):
1977                         isAnnotated = True
1978                     elif re.match(r'\s*$', l):
1979                         inHeader = False
1980                         continue
1981                 else:
1982                     body.append(l)
1983
1984             if not isAnnotated:
1985                 body = ["lightweight tag imported by git p4\n"]
1986
1987             # Create the label - use the same view as the client spec we are using
1988             clientSpec = getClientSpec()
1989
1990             labelTemplate  = "Label: %s\n" % name
1991             labelTemplate += "Description:\n"
1992             for b in body:
1993                 labelTemplate += "\t" + b + "\n"
1994             labelTemplate += "View:\n"
1995             for depot_side in clientSpec.mappings:
1996                 labelTemplate += "\t%s\n" % depot_side
1997
1998             if self.dry_run:
1999                 print "Would create p4 label %s for tag" % name
2000             elif self.prepare_p4_only:
2001                 print "Not creating p4 label %s for tag due to option" \
2002                       " --prepare-p4-only" % name
2003             else:
2004                 p4_write_pipe(["label", "-i"], labelTemplate)
2005
2006                 # Use the label
2007                 p4_system(["tag", "-l", name] +
2008                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2009
2010                 if verbose:
2011                     print "created p4 label for tag %s" % name
2012
2013     def run(self, args):
2014         if len(args) == 0:
2015             self.master = currentGitBranch()
2016         elif len(args) == 1:
2017             self.master = args[0]
2018             if not branchExists(self.master):
2019                 die("Branch %s does not exist" % self.master)
2020         else:
2021             return False
2022
2023         for i in self.update_shelve:
2024             if i <= 0:
2025                 sys.exit("invalid changelist %d" % i)
2026
2027         if self.master:
2028             allowSubmit = gitConfig("git-p4.allowSubmit")
2029             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2030                 die("%s is not in git-p4.allowSubmit" % self.master)
2031
2032         [upstream, settings] = findUpstreamBranchPoint()
2033         self.depotPath = settings['depot-paths'][0]
2034         if len(self.origin) == 0:
2035             self.origin = upstream
2036
2037         if len(self.update_shelve) > 0:
2038             self.shelve = True
2039
2040         if self.preserveUser:
2041             if not self.canChangeChangelists():
2042                 die("Cannot preserve user names without p4 super-user or admin permissions")
2043
2044         # if not set from the command line, try the config file
2045         if self.conflict_behavior is None:
2046             val = gitConfig("git-p4.conflict")
2047             if val:
2048                 if val not in self.conflict_behavior_choices:
2049                     die("Invalid value '%s' for config git-p4.conflict" % val)
2050             else:
2051                 val = "ask"
2052             self.conflict_behavior = val
2053
2054         if self.verbose:
2055             print "Origin branch is " + self.origin
2056
2057         if len(self.depotPath) == 0:
2058             print "Internal error: cannot locate perforce depot path from existing branches"
2059             sys.exit(128)
2060
2061         self.useClientSpec = False
2062         if gitConfigBool("git-p4.useclientspec"):
2063             self.useClientSpec = True
2064         if self.useClientSpec:
2065             self.clientSpecDirs = getClientSpec()
2066
2067         # Check for the existence of P4 branches
2068         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2069
2070         if self.useClientSpec and not branchesDetected:
2071             # all files are relative to the client spec
2072             self.clientPath = getClientRoot()
2073         else:
2074             self.clientPath = p4Where(self.depotPath)
2075
2076         if self.clientPath == "":
2077             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2078
2079         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2080         self.oldWorkingDirectory = os.getcwd()
2081
2082         # ensure the clientPath exists
2083         new_client_dir = False
2084         if not os.path.exists(self.clientPath):
2085             new_client_dir = True
2086             os.makedirs(self.clientPath)
2087
2088         chdir(self.clientPath, is_client_path=True)
2089         if self.dry_run:
2090             print "Would synchronize p4 checkout in %s" % self.clientPath
2091         else:
2092             print "Synchronizing p4 checkout..."
2093             if new_client_dir:
2094                 # old one was destroyed, and maybe nobody told p4
2095                 p4_sync("...", "-f")
2096             else:
2097                 p4_sync("...")
2098         self.check()
2099
2100         commits = []
2101         if self.master:
2102             commitish = self.master
2103         else:
2104             commitish = 'HEAD'
2105
2106         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2107             commits.append(line.strip())
2108         commits.reverse()
2109
2110         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2111             self.checkAuthorship = False
2112         else:
2113             self.checkAuthorship = True
2114
2115         if self.preserveUser:
2116             self.checkValidP4Users(commits)
2117
2118         #
2119         # Build up a set of options to be passed to diff when
2120         # submitting each commit to p4.
2121         #
2122         if self.detectRenames:
2123             # command-line -M arg
2124             self.diffOpts = "-M"
2125         else:
2126             # If not explicitly set check the config variable
2127             detectRenames = gitConfig("git-p4.detectRenames")
2128
2129             if detectRenames.lower() == "false" or detectRenames == "":
2130                 self.diffOpts = ""
2131             elif detectRenames.lower() == "true":
2132                 self.diffOpts = "-M"
2133             else:
2134                 self.diffOpts = "-M%s" % detectRenames
2135
2136         # no command-line arg for -C or --find-copies-harder, just
2137         # config variables
2138         detectCopies = gitConfig("git-p4.detectCopies")
2139         if detectCopies.lower() == "false" or detectCopies == "":
2140             pass
2141         elif detectCopies.lower() == "true":
2142             self.diffOpts += " -C"
2143         else:
2144             self.diffOpts += " -C%s" % detectCopies
2145
2146         if gitConfigBool("git-p4.detectCopiesHarder"):
2147             self.diffOpts += " --find-copies-harder"
2148
2149         num_shelves = len(self.update_shelve)
2150         if num_shelves > 0 and num_shelves != len(commits):
2151             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2152                      (len(commits), num_shelves))
2153
2154         #
2155         # Apply the commits, one at a time.  On failure, ask if should
2156         # continue to try the rest of the patches, or quit.
2157         #
2158         if self.dry_run:
2159             print "Would apply"
2160         applied = []
2161         last = len(commits) - 1
2162         for i, commit in enumerate(commits):
2163             if self.dry_run:
2164                 print " ", read_pipe(["git", "show", "-s",
2165                                       "--format=format:%h %s", commit])
2166                 ok = True
2167             else:
2168                 ok = self.applyCommit(commit)
2169             if ok:
2170                 applied.append(commit)
2171             else:
2172                 if self.prepare_p4_only and i < last:
2173                     print "Processing only the first commit due to option" \
2174                           " --prepare-p4-only"
2175                     break
2176                 if i < last:
2177                     quit = False
2178                     while True:
2179                         # prompt for what to do, or use the option/variable
2180                         if self.conflict_behavior == "ask":
2181                             print "What do you want to do?"
2182                             response = raw_input("[s]kip this commit but apply"
2183                                                  " the rest, or [q]uit? ")
2184                             if not response:
2185                                 continue
2186                         elif self.conflict_behavior == "skip":
2187                             response = "s"
2188                         elif self.conflict_behavior == "quit":
2189                             response = "q"
2190                         else:
2191                             die("Unknown conflict_behavior '%s'" %
2192                                 self.conflict_behavior)
2193
2194                         if response[0] == "s":
2195                             print "Skipping this commit, but applying the rest"
2196                             break
2197                         if response[0] == "q":
2198                             print "Quitting"
2199                             quit = True
2200                             break
2201                     if quit:
2202                         break
2203
2204         chdir(self.oldWorkingDirectory)
2205         shelved_applied = "shelved" if self.shelve else "applied"
2206         if self.dry_run:
2207             pass
2208         elif self.prepare_p4_only:
2209             pass
2210         elif len(commits) == len(applied):
2211             print ("All commits {0}!".format(shelved_applied))
2212
2213             sync = P4Sync()
2214             if self.branch:
2215                 sync.branch = self.branch
2216             sync.run([])
2217
2218             rebase = P4Rebase()
2219             rebase.rebase()
2220
2221         else:
2222             if len(applied) == 0:
2223                 print ("No commits {0}.".format(shelved_applied))
2224             else:
2225                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2226                 for c in commits:
2227                     if c in applied:
2228                         star = "*"
2229                     else:
2230                         star = " "
2231                     print star, read_pipe(["git", "show", "-s",
2232                                            "--format=format:%h %s",  c])
2233                 print "You will have to do 'git p4 sync' and rebase."
2234
2235         if gitConfigBool("git-p4.exportLabels"):
2236             self.exportLabels = True
2237
2238         if self.exportLabels:
2239             p4Labels = getP4Labels(self.depotPath)
2240             gitTags = getGitTags()
2241
2242             missingGitTags = gitTags - p4Labels
2243             self.exportGitTags(missingGitTags)
2244
2245         # exit with error unless everything applied perfectly
2246         if len(commits) != len(applied):
2247                 sys.exit(1)
2248
2249         return True
2250
2251 class View(object):
2252     """Represent a p4 view ("p4 help views"), and map files in a
2253        repo according to the view."""
2254
2255     def __init__(self, client_name):
2256         self.mappings = []
2257         self.client_prefix = "//%s/" % client_name
2258         # cache results of "p4 where" to lookup client file locations
2259         self.client_spec_path_cache = {}
2260
2261     def append(self, view_line):
2262         """Parse a view line, splitting it into depot and client
2263            sides.  Append to self.mappings, preserving order.  This
2264            is only needed for tag creation."""
2265
2266         # Split the view line into exactly two words.  P4 enforces
2267         # structure on these lines that simplifies this quite a bit.
2268         #
2269         # Either or both words may be double-quoted.
2270         # Single quotes do not matter.
2271         # Double-quote marks cannot occur inside the words.
2272         # A + or - prefix is also inside the quotes.
2273         # There are no quotes unless they contain a space.
2274         # The line is already white-space stripped.
2275         # The two words are separated by a single space.
2276         #
2277         if view_line[0] == '"':
2278             # First word is double quoted.  Find its end.
2279             close_quote_index = view_line.find('"', 1)
2280             if close_quote_index <= 0:
2281                 die("No first-word closing quote found: %s" % view_line)
2282             depot_side = view_line[1:close_quote_index]
2283             # skip closing quote and space
2284             rhs_index = close_quote_index + 1 + 1
2285         else:
2286             space_index = view_line.find(" ")
2287             if space_index <= 0:
2288                 die("No word-splitting space found: %s" % view_line)
2289             depot_side = view_line[0:space_index]
2290             rhs_index = space_index + 1
2291
2292         # prefix + means overlay on previous mapping
2293         if depot_side.startswith("+"):
2294             depot_side = depot_side[1:]
2295
2296         # prefix - means exclude this path, leave out of mappings
2297         exclude = False
2298         if depot_side.startswith("-"):
2299             exclude = True
2300             depot_side = depot_side[1:]
2301
2302         if not exclude:
2303             self.mappings.append(depot_side)
2304
2305     def convert_client_path(self, clientFile):
2306         # chop off //client/ part to make it relative
2307         if not clientFile.startswith(self.client_prefix):
2308             die("No prefix '%s' on clientFile '%s'" %
2309                 (self.client_prefix, clientFile))
2310         return clientFile[len(self.client_prefix):]
2311
2312     def update_client_spec_path_cache(self, files):
2313         """ Caching file paths by "p4 where" batch query """
2314
2315         # List depot file paths exclude that already cached
2316         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2317
2318         if len(fileArgs) == 0:
2319             return  # All files in cache
2320
2321         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2322         for res in where_result:
2323             if "code" in res and res["code"] == "error":
2324                 # assume error is "... file(s) not in client view"
2325                 continue
2326             if "clientFile" not in res:
2327                 die("No clientFile in 'p4 where' output")
2328             if "unmap" in res:
2329                 # it will list all of them, but only one not unmap-ped
2330                 continue
2331             if gitConfigBool("core.ignorecase"):
2332                 res['depotFile'] = res['depotFile'].lower()
2333             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2334
2335         # not found files or unmap files set to ""
2336         for depotFile in fileArgs:
2337             if gitConfigBool("core.ignorecase"):
2338                 depotFile = depotFile.lower()
2339             if depotFile not in self.client_spec_path_cache:
2340                 self.client_spec_path_cache[depotFile] = ""
2341
2342     def map_in_client(self, depot_path):
2343         """Return the relative location in the client where this
2344            depot file should live.  Returns "" if the file should
2345            not be mapped in the client."""
2346
2347         if gitConfigBool("core.ignorecase"):
2348             depot_path = depot_path.lower()
2349
2350         if depot_path in self.client_spec_path_cache:
2351             return self.client_spec_path_cache[depot_path]
2352
2353         die( "Error: %s is not found in client spec path" % depot_path )
2354         return ""
2355
2356 class P4Sync(Command, P4UserMap):
2357     delete_actions = ( "delete", "move/delete", "purge" )
2358
2359     def __init__(self):
2360         Command.__init__(self)
2361         P4UserMap.__init__(self)
2362         self.options = [
2363                 optparse.make_option("--branch", dest="branch"),
2364                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2365                 optparse.make_option("--changesfile", dest="changesFile"),
2366                 optparse.make_option("--silent", dest="silent", action="store_true"),
2367                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2368                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2369                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2370                                      help="Import into refs/heads/ , not refs/remotes"),
2371                 optparse.make_option("--max-changes", dest="maxChanges",
2372                                      help="Maximum number of changes to import"),
2373                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2374                                      help="Internal block size to use when iteratively calling p4 changes"),
2375                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2376                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2377                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2378                                      help="Only sync files that are included in the Perforce Client Spec"),
2379                 optparse.make_option("-/", dest="cloneExclude",
2380                                      action="append", type="string",
2381                                      help="exclude depot path"),
2382         ]
2383         self.description = """Imports from Perforce into a git repository.\n
2384     example:
2385     //depot/my/project/ -- to import the current head
2386     //depot/my/project/@all -- to import everything
2387     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2388
2389     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2390
2391         self.usage += " //depot/path[@revRange]"
2392         self.silent = False
2393         self.createdBranches = set()
2394         self.committedChanges = set()
2395         self.branch = ""
2396         self.detectBranches = False
2397         self.detectLabels = False
2398         self.importLabels = False
2399         self.changesFile = ""
2400         self.syncWithOrigin = True
2401         self.importIntoRemotes = True
2402         self.maxChanges = ""
2403         self.changes_block_size = None
2404         self.keepRepoPath = False
2405         self.depotPaths = None
2406         self.p4BranchesInGit = []
2407         self.cloneExclude = []
2408         self.useClientSpec = False
2409         self.useClientSpec_from_options = False
2410         self.clientSpecDirs = None
2411         self.tempBranches = []
2412         self.tempBranchLocation = "refs/git-p4-tmp"
2413         self.largeFileSystem = None
2414
2415         if gitConfig('git-p4.largeFileSystem'):
2416             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2417             self.largeFileSystem = largeFileSystemConstructor(
2418                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2419             )
2420
2421         if gitConfig("git-p4.syncFromOrigin") == "false":
2422             self.syncWithOrigin = False
2423
2424     # Force a checkpoint in fast-import and wait for it to finish
2425     def checkpoint(self):
2426         self.gitStream.write("checkpoint\n\n")
2427         self.gitStream.write("progress checkpoint\n\n")
2428         out = self.gitOutput.readline()
2429         if self.verbose:
2430             print "checkpoint finished: " + out
2431
2432     def extractFilesFromCommit(self, commit):
2433         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2434                              for path in self.cloneExclude]
2435         files = []
2436         fnum = 0
2437         while commit.has_key("depotFile%s" % fnum):
2438             path =  commit["depotFile%s" % fnum]
2439
2440             if [p for p in self.cloneExclude
2441                 if p4PathStartsWith(path, p)]:
2442                 found = False
2443             else:
2444                 found = [p for p in self.depotPaths
2445                          if p4PathStartsWith(path, p)]
2446             if not found:
2447                 fnum = fnum + 1
2448                 continue
2449
2450             file = {}
2451             file["path"] = path
2452             file["rev"] = commit["rev%s" % fnum]
2453             file["action"] = commit["action%s" % fnum]
2454             file["type"] = commit["type%s" % fnum]
2455             files.append(file)
2456             fnum = fnum + 1
2457         return files
2458
2459     def extractJobsFromCommit(self, commit):
2460         jobs = []
2461         jnum = 0
2462         while commit.has_key("job%s" % jnum):
2463             job = commit["job%s" % jnum]
2464             jobs.append(job)
2465             jnum = jnum + 1
2466         return jobs
2467
2468     def stripRepoPath(self, path, prefixes):
2469         """When streaming files, this is called to map a p4 depot path
2470            to where it should go in git.  The prefixes are either
2471            self.depotPaths, or self.branchPrefixes in the case of
2472            branch detection."""
2473
2474         if self.useClientSpec:
2475             # branch detection moves files up a level (the branch name)
2476             # from what client spec interpretation gives
2477             path = self.clientSpecDirs.map_in_client(path)
2478             if self.detectBranches:
2479                 for b in self.knownBranches:
2480                     if path.startswith(b + "/"):
2481                         path = path[len(b)+1:]
2482
2483         elif self.keepRepoPath:
2484             # Preserve everything in relative path name except leading
2485             # //depot/; just look at first prefix as they all should
2486             # be in the same depot.
2487             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2488             if p4PathStartsWith(path, depot):
2489                 path = path[len(depot):]
2490
2491         else:
2492             for p in prefixes:
2493                 if p4PathStartsWith(path, p):
2494                     path = path[len(p):]
2495                     break
2496
2497         path = wildcard_decode(path)
2498         return path
2499
2500     def splitFilesIntoBranches(self, commit):
2501         """Look at each depotFile in the commit to figure out to what
2502            branch it belongs."""
2503
2504         if self.clientSpecDirs:
2505             files = self.extractFilesFromCommit(commit)
2506             self.clientSpecDirs.update_client_spec_path_cache(files)
2507
2508         branches = {}
2509         fnum = 0
2510         while commit.has_key("depotFile%s" % fnum):
2511             path =  commit["depotFile%s" % fnum]
2512             found = [p for p in self.depotPaths
2513                      if p4PathStartsWith(path, p)]
2514             if not found:
2515                 fnum = fnum + 1
2516                 continue
2517
2518             file = {}
2519             file["path"] = path
2520             file["rev"] = commit["rev%s" % fnum]
2521             file["action"] = commit["action%s" % fnum]
2522             file["type"] = commit["type%s" % fnum]
2523             fnum = fnum + 1
2524
2525             # start with the full relative path where this file would
2526             # go in a p4 client
2527             if self.useClientSpec:
2528                 relPath = self.clientSpecDirs.map_in_client(path)
2529             else:
2530                 relPath = self.stripRepoPath(path, self.depotPaths)
2531
2532             for branch in self.knownBranches.keys():
2533                 # add a trailing slash so that a commit into qt/4.2foo
2534                 # doesn't end up in qt/4.2, e.g.
2535                 if relPath.startswith(branch + "/"):
2536                     if branch not in branches:
2537                         branches[branch] = []
2538                     branches[branch].append(file)
2539                     break
2540
2541         return branches
2542
2543     def writeToGitStream(self, gitMode, relPath, contents):
2544         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2545         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2546         for d in contents:
2547             self.gitStream.write(d)
2548         self.gitStream.write('\n')
2549
2550     def encodeWithUTF8(self, path):
2551         try:
2552             path.decode('ascii')
2553         except:
2554             encoding = 'utf8'
2555             if gitConfig('git-p4.pathEncoding'):
2556                 encoding = gitConfig('git-p4.pathEncoding')
2557             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2558             if self.verbose:
2559                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2560         return path
2561
2562     # output one file from the P4 stream
2563     # - helper for streamP4Files
2564
2565     def streamOneP4File(self, file, contents):
2566         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2567         relPath = self.encodeWithUTF8(relPath)
2568         if verbose:
2569             size = int(self.stream_file['fileSize'])
2570             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2571             sys.stdout.flush()
2572
2573         (type_base, type_mods) = split_p4_type(file["type"])
2574
2575         git_mode = "100644"
2576         if "x" in type_mods:
2577             git_mode = "100755"
2578         if type_base == "symlink":
2579             git_mode = "120000"
2580             # p4 print on a symlink sometimes contains "target\n";
2581             # if it does, remove the newline
2582             data = ''.join(contents)
2583             if not data:
2584                 # Some version of p4 allowed creating a symlink that pointed
2585                 # to nothing.  This causes p4 errors when checking out such
2586                 # a change, and errors here too.  Work around it by ignoring
2587                 # the bad symlink; hopefully a future change fixes it.
2588                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2589                 return
2590             elif data[-1] == '\n':
2591                 contents = [data[:-1]]
2592             else:
2593                 contents = [data]
2594
2595         if type_base == "utf16":
2596             # p4 delivers different text in the python output to -G
2597             # than it does when using "print -o", or normal p4 client
2598             # operations.  utf16 is converted to ascii or utf8, perhaps.
2599             # But ascii text saved as -t utf16 is completely mangled.
2600             # Invoke print -o to get the real contents.
2601             #
2602             # On windows, the newlines will always be mangled by print, so put
2603             # them back too.  This is not needed to the cygwin windows version,
2604             # just the native "NT" type.
2605             #
2606             try:
2607                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2608             except Exception as e:
2609                 if 'Translation of file content failed' in str(e):
2610                     type_base = 'binary'
2611                 else:
2612                     raise e
2613             else:
2614                 if p4_version_string().find('/NT') >= 0:
2615                     text = text.replace('\r\n', '\n')
2616                 contents = [ text ]
2617
2618         if type_base == "apple":
2619             # Apple filetype files will be streamed as a concatenation of
2620             # its appledouble header and the contents.  This is useless
2621             # on both macs and non-macs.  If using "print -q -o xx", it
2622             # will create "xx" with the data, and "%xx" with the header.
2623             # This is also not very useful.
2624             #
2625             # Ideally, someday, this script can learn how to generate
2626             # appledouble files directly and import those to git, but
2627             # non-mac machines can never find a use for apple filetype.
2628             print "\nIgnoring apple filetype file %s" % file['depotFile']
2629             return
2630
2631         # Note that we do not try to de-mangle keywords on utf16 files,
2632         # even though in theory somebody may want that.
2633         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2634         if pattern:
2635             regexp = re.compile(pattern, re.VERBOSE)
2636             text = ''.join(contents)
2637             text = regexp.sub(r'$\1$', text)
2638             contents = [ text ]
2639
2640         if self.largeFileSystem:
2641             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2642
2643         self.writeToGitStream(git_mode, relPath, contents)
2644
2645     def streamOneP4Deletion(self, file):
2646         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2647         relPath = self.encodeWithUTF8(relPath)
2648         if verbose:
2649             sys.stdout.write("delete %s\n" % relPath)
2650             sys.stdout.flush()
2651         self.gitStream.write("D %s\n" % relPath)
2652
2653         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2654             self.largeFileSystem.removeLargeFile(relPath)
2655
2656     # handle another chunk of streaming data
2657     def streamP4FilesCb(self, marshalled):
2658
2659         # catch p4 errors and complain
2660         err = None
2661         if "code" in marshalled:
2662             if marshalled["code"] == "error":
2663                 if "data" in marshalled:
2664                     err = marshalled["data"].rstrip()
2665
2666         if not err and 'fileSize' in self.stream_file:
2667             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2668             if required_bytes > 0:
2669                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2670                     os.getcwd(), required_bytes/1024/1024
2671                 )
2672
2673         if err:
2674             f = None
2675             if self.stream_have_file_info:
2676                 if "depotFile" in self.stream_file:
2677                     f = self.stream_file["depotFile"]
2678             # force a failure in fast-import, else an empty
2679             # commit will be made
2680             self.gitStream.write("\n")
2681             self.gitStream.write("die-now\n")
2682             self.gitStream.close()
2683             # ignore errors, but make sure it exits first
2684             self.importProcess.wait()
2685             if f:
2686                 die("Error from p4 print for %s: %s" % (f, err))
2687             else:
2688                 die("Error from p4 print: %s" % err)
2689
2690         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2691             # start of a new file - output the old one first
2692             self.streamOneP4File(self.stream_file, self.stream_contents)
2693             self.stream_file = {}
2694             self.stream_contents = []
2695             self.stream_have_file_info = False
2696
2697         # pick up the new file information... for the
2698         # 'data' field we need to append to our array
2699         for k in marshalled.keys():
2700             if k == 'data':
2701                 if 'streamContentSize' not in self.stream_file:
2702                     self.stream_file['streamContentSize'] = 0
2703                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2704                 self.stream_contents.append(marshalled['data'])
2705             else:
2706                 self.stream_file[k] = marshalled[k]
2707
2708         if (verbose and
2709             'streamContentSize' in self.stream_file and
2710             'fileSize' in self.stream_file and
2711             'depotFile' in self.stream_file):
2712             size = int(self.stream_file["fileSize"])
2713             if size > 0:
2714                 progress = 100*self.stream_file['streamContentSize']/size
2715                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2716                 sys.stdout.flush()
2717
2718         self.stream_have_file_info = True
2719
2720     # Stream directly from "p4 files" into "git fast-import"
2721     def streamP4Files(self, files):
2722         filesForCommit = []
2723         filesToRead = []
2724         filesToDelete = []
2725
2726         for f in files:
2727             filesForCommit.append(f)
2728             if f['action'] in self.delete_actions:
2729                 filesToDelete.append(f)
2730             else:
2731                 filesToRead.append(f)
2732
2733         # deleted files...
2734         for f in filesToDelete:
2735             self.streamOneP4Deletion(f)
2736
2737         if len(filesToRead) > 0:
2738             self.stream_file = {}
2739             self.stream_contents = []
2740             self.stream_have_file_info = False
2741
2742             # curry self argument
2743             def streamP4FilesCbSelf(entry):
2744                 self.streamP4FilesCb(entry)
2745
2746             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2747
2748             p4CmdList(["-x", "-", "print"],
2749                       stdin=fileArgs,
2750                       cb=streamP4FilesCbSelf)
2751
2752             # do the last chunk
2753             if self.stream_file.has_key('depotFile'):
2754                 self.streamOneP4File(self.stream_file, self.stream_contents)
2755
2756     def make_email(self, userid):
2757         if userid in self.users:
2758             return self.users[userid]
2759         else:
2760             return "%s <a@b>" % userid
2761
2762     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2763         """ Stream a p4 tag.
2764         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2765         """
2766
2767         if verbose:
2768             print "writing tag %s for commit %s" % (labelName, commit)
2769         gitStream.write("tag %s\n" % labelName)
2770         gitStream.write("from %s\n" % commit)
2771
2772         if labelDetails.has_key('Owner'):
2773             owner = labelDetails["Owner"]
2774         else:
2775             owner = None
2776
2777         # Try to use the owner of the p4 label, or failing that,
2778         # the current p4 user id.
2779         if owner:
2780             email = self.make_email(owner)
2781         else:
2782             email = self.make_email(self.p4UserId())
2783         tagger = "%s %s %s" % (email, epoch, self.tz)
2784
2785         gitStream.write("tagger %s\n" % tagger)
2786
2787         print "labelDetails=",labelDetails
2788         if labelDetails.has_key('Description'):
2789             description = labelDetails['Description']
2790         else:
2791             description = 'Label from git p4'
2792
2793         gitStream.write("data %d\n" % len(description))
2794         gitStream.write(description)
2795         gitStream.write("\n")
2796
2797     def inClientSpec(self, path):
2798         if not self.clientSpecDirs:
2799             return True
2800         inClientSpec = self.clientSpecDirs.map_in_client(path)
2801         if not inClientSpec and self.verbose:
2802             print('Ignoring file outside of client spec: {0}'.format(path))
2803         return inClientSpec
2804
2805     def hasBranchPrefix(self, path):
2806         if not self.branchPrefixes:
2807             return True
2808         hasPrefix = [p for p in self.branchPrefixes
2809                         if p4PathStartsWith(path, p)]
2810         if not hasPrefix and self.verbose:
2811             print('Ignoring file outside of prefix: {0}'.format(path))
2812         return hasPrefix
2813
2814     def commit(self, details, files, branch, parent = ""):
2815         epoch = details["time"]
2816         author = details["user"]
2817         jobs = self.extractJobsFromCommit(details)
2818
2819         if self.verbose:
2820             print('commit into {0}'.format(branch))
2821
2822         if self.clientSpecDirs:
2823             self.clientSpecDirs.update_client_spec_path_cache(files)
2824
2825         files = [f for f in files
2826             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2827
2828         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2829             print('Ignoring revision {0} as it would produce an empty commit.'
2830                 .format(details['change']))
2831             return
2832
2833         self.gitStream.write("commit %s\n" % branch)
2834         self.gitStream.write("mark :%s\n" % details["change"])
2835         self.committedChanges.add(int(details["change"]))
2836         committer = ""
2837         if author not in self.users:
2838             self.getUserMapFromPerforceServer()
2839         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2840
2841         self.gitStream.write("committer %s\n" % committer)
2842
2843         self.gitStream.write("data <<EOT\n")
2844         self.gitStream.write(details["desc"])
2845         if len(jobs) > 0:
2846             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2847         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2848                              (','.join(self.branchPrefixes), details["change"]))
2849         if len(details['options']) > 0:
2850             self.gitStream.write(": options = %s" % details['options'])
2851         self.gitStream.write("]\nEOT\n\n")
2852
2853         if len(parent) > 0:
2854             if self.verbose:
2855                 print "parent %s" % parent
2856             self.gitStream.write("from %s\n" % parent)
2857
2858         self.streamP4Files(files)
2859         self.gitStream.write("\n")
2860
2861         change = int(details["change"])
2862
2863         if self.labels.has_key(change):
2864             label = self.labels[change]
2865             labelDetails = label[0]
2866             labelRevisions = label[1]
2867             if self.verbose:
2868                 print "Change %s is labelled %s" % (change, labelDetails)
2869
2870             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2871                                                 for p in self.branchPrefixes])
2872
2873             if len(files) == len(labelRevisions):
2874
2875                 cleanedFiles = {}
2876                 for info in files:
2877                     if info["action"] in self.delete_actions:
2878                         continue
2879                     cleanedFiles[info["depotFile"]] = info["rev"]
2880
2881                 if cleanedFiles == labelRevisions:
2882                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2883
2884                 else:
2885                     if not self.silent:
2886                         print ("Tag %s does not match with change %s: files do not match."
2887                                % (labelDetails["label"], change))
2888
2889             else:
2890                 if not self.silent:
2891                     print ("Tag %s does not match with change %s: file count is different."
2892                            % (labelDetails["label"], change))
2893
2894     # Build a dictionary of changelists and labels, for "detect-labels" option.
2895     def getLabels(self):
2896         self.labels = {}
2897
2898         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2899         if len(l) > 0 and not self.silent:
2900             print "Finding files belonging to labels in %s" % `self.depotPaths`
2901
2902         for output in l:
2903             label = output["label"]
2904             revisions = {}
2905             newestChange = 0
2906             if self.verbose:
2907                 print "Querying files for label %s" % label
2908             for file in p4CmdList(["files"] +
2909                                       ["%s...@%s" % (p, label)
2910                                           for p in self.depotPaths]):
2911                 revisions[file["depotFile"]] = file["rev"]
2912                 change = int(file["change"])
2913                 if change > newestChange:
2914                     newestChange = change
2915
2916             self.labels[newestChange] = [output, revisions]
2917
2918         if self.verbose:
2919             print "Label changes: %s" % self.labels.keys()
2920
2921     # Import p4 labels as git tags. A direct mapping does not
2922     # exist, so assume that if all the files are at the same revision
2923     # then we can use that, or it's something more complicated we should
2924     # just ignore.
2925     def importP4Labels(self, stream, p4Labels):
2926         if verbose:
2927             print "import p4 labels: " + ' '.join(p4Labels)
2928
2929         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2930         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2931         if len(validLabelRegexp) == 0:
2932             validLabelRegexp = defaultLabelRegexp
2933         m = re.compile(validLabelRegexp)
2934
2935         for name in p4Labels:
2936             commitFound = False
2937
2938             if not m.match(name):
2939                 if verbose:
2940                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2941                 continue
2942
2943             if name in ignoredP4Labels:
2944                 continue
2945
2946             labelDetails = p4CmdList(['label', "-o", name])[0]
2947
2948             # get the most recent changelist for each file in this label
2949             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2950                                 for p in self.depotPaths])
2951
2952             if change.has_key('change'):
2953                 # find the corresponding git commit; take the oldest commit
2954                 changelist = int(change['change'])
2955                 if changelist in self.committedChanges:
2956                     gitCommit = ":%d" % changelist       # use a fast-import mark
2957                     commitFound = True
2958                 else:
2959                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2960                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2961                     if len(gitCommit) == 0:
2962                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2963                     else:
2964                         commitFound = True
2965                         gitCommit = gitCommit.strip()
2966
2967                 if commitFound:
2968                     # Convert from p4 time format
2969                     try:
2970                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2971                     except ValueError:
2972                         print "Could not convert label time %s" % labelDetails['Update']
2973                         tmwhen = 1
2974
2975                     when = int(time.mktime(tmwhen))
2976                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2977                     if verbose:
2978                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2979             else:
2980                 if verbose:
2981                     print "Label %s has no changelists - possibly deleted?" % name
2982
2983             if not commitFound:
2984                 # We can't import this label; don't try again as it will get very
2985                 # expensive repeatedly fetching all the files for labels that will
2986                 # never be imported. If the label is moved in the future, the
2987                 # ignore will need to be removed manually.
2988                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2989
2990     def guessProjectName(self):
2991         for p in self.depotPaths:
2992             if p.endswith("/"):
2993                 p = p[:-1]
2994             p = p[p.strip().rfind("/") + 1:]
2995             if not p.endswith("/"):
2996                p += "/"
2997             return p
2998
2999     def getBranchMapping(self):
3000         lostAndFoundBranches = set()
3001
3002         user = gitConfig("git-p4.branchUser")
3003         if len(user) > 0:
3004             command = "branches -u %s" % user
3005         else:
3006             command = "branches"
3007
3008         for info in p4CmdList(command):
3009             details = p4Cmd(["branch", "-o", info["branch"]])
3010             viewIdx = 0
3011             while details.has_key("View%s" % viewIdx):
3012                 paths = details["View%s" % viewIdx].split(" ")
3013                 viewIdx = viewIdx + 1
3014                 # require standard //depot/foo/... //depot/bar/... mapping
3015                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3016                     continue
3017                 source = paths[0]
3018                 destination = paths[1]
3019                 ## HACK
3020                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3021                     source = source[len(self.depotPaths[0]):-4]
3022                     destination = destination[len(self.depotPaths[0]):-4]
3023
3024                     if destination in self.knownBranches:
3025                         if not self.silent:
3026                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3027                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3028                         continue
3029
3030                     self.knownBranches[destination] = source
3031
3032                     lostAndFoundBranches.discard(destination)
3033
3034                     if source not in self.knownBranches:
3035                         lostAndFoundBranches.add(source)
3036
3037         # Perforce does not strictly require branches to be defined, so we also
3038         # check git config for a branch list.
3039         #
3040         # Example of branch definition in git config file:
3041         # [git-p4]
3042         #   branchList=main:branchA
3043         #   branchList=main:branchB
3044         #   branchList=branchA:branchC
3045         configBranches = gitConfigList("git-p4.branchList")
3046         for branch in configBranches:
3047             if branch:
3048                 (source, destination) = branch.split(":")
3049                 self.knownBranches[destination] = source
3050
3051                 lostAndFoundBranches.discard(destination)
3052
3053                 if source not in self.knownBranches:
3054                     lostAndFoundBranches.add(source)
3055
3056
3057         for branch in lostAndFoundBranches:
3058             self.knownBranches[branch] = branch
3059
3060     def getBranchMappingFromGitBranches(self):
3061         branches = p4BranchesInGit(self.importIntoRemotes)
3062         for branch in branches.keys():
3063             if branch == "master":
3064                 branch = "main"
3065             else:
3066                 branch = branch[len(self.projectName):]
3067             self.knownBranches[branch] = branch
3068
3069     def updateOptionDict(self, d):
3070         option_keys = {}
3071         if self.keepRepoPath:
3072             option_keys['keepRepoPath'] = 1
3073
3074         d["options"] = ' '.join(sorted(option_keys.keys()))
3075
3076     def readOptions(self, d):
3077         self.keepRepoPath = (d.has_key('options')
3078                              and ('keepRepoPath' in d['options']))
3079
3080     def gitRefForBranch(self, branch):
3081         if branch == "main":
3082             return self.refPrefix + "master"
3083
3084         if len(branch) <= 0:
3085             return branch
3086
3087         return self.refPrefix + self.projectName + branch
3088
3089     def gitCommitByP4Change(self, ref, change):
3090         if self.verbose:
3091             print "looking in ref " + ref + " for change %s using bisect..." % change
3092
3093         earliestCommit = ""
3094         latestCommit = parseRevision(ref)
3095
3096         while True:
3097             if self.verbose:
3098                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3099             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3100             if len(next) == 0:
3101                 if self.verbose:
3102                     print "argh"
3103                 return ""
3104             log = extractLogMessageFromGitCommit(next)
3105             settings = extractSettingsGitLog(log)
3106             currentChange = int(settings['change'])
3107             if self.verbose:
3108                 print "current change %s" % currentChange
3109
3110             if currentChange == change:
3111                 if self.verbose:
3112                     print "found %s" % next
3113                 return next
3114
3115             if currentChange < change:
3116                 earliestCommit = "^%s" % next
3117             else:
3118                 latestCommit = "%s" % next
3119
3120         return ""
3121
3122     def importNewBranch(self, branch, maxChange):
3123         # make fast-import flush all changes to disk and update the refs using the checkpoint
3124         # command so that we can try to find the branch parent in the git history
3125         self.gitStream.write("checkpoint\n\n");
3126         self.gitStream.flush();
3127         branchPrefix = self.depotPaths[0] + branch + "/"
3128         range = "@1,%s" % maxChange
3129         #print "prefix" + branchPrefix
3130         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3131         if len(changes) <= 0:
3132             return False
3133         firstChange = changes[0]
3134         #print "first change in branch: %s" % firstChange
3135         sourceBranch = self.knownBranches[branch]
3136         sourceDepotPath = self.depotPaths[0] + sourceBranch
3137         sourceRef = self.gitRefForBranch(sourceBranch)
3138         #print "source " + sourceBranch
3139
3140         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3141         #print "branch parent: %s" % branchParentChange
3142         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3143         if len(gitParent) > 0:
3144             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3145             #print "parent git commit: %s" % gitParent
3146
3147         self.importChanges(changes)
3148         return True
3149
3150     def searchParent(self, parent, branch, target):
3151         parentFound = False
3152         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3153                                      "--no-merges", parent]):
3154             blob = blob.strip()
3155             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3156                 parentFound = True
3157                 if self.verbose:
3158                     print "Found parent of %s in commit %s" % (branch, blob)
3159                 break
3160         if parentFound:
3161             return blob
3162         else:
3163             return None
3164
3165     def importChanges(self, changes):
3166         cnt = 1
3167         for change in changes:
3168             description = p4_describe(change)
3169             self.updateOptionDict(description)
3170
3171             if not self.silent:
3172                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3173                 sys.stdout.flush()
3174             cnt = cnt + 1
3175
3176             try:
3177                 if self.detectBranches:
3178                     branches = self.splitFilesIntoBranches(description)
3179                     for branch in branches.keys():
3180                         ## HACK  --hwn
3181                         branchPrefix = self.depotPaths[0] + branch + "/"
3182                         self.branchPrefixes = [ branchPrefix ]
3183
3184                         parent = ""
3185
3186                         filesForCommit = branches[branch]
3187
3188                         if self.verbose:
3189                             print "branch is %s" % branch
3190
3191                         self.updatedBranches.add(branch)
3192
3193                         if branch not in self.createdBranches:
3194                             self.createdBranches.add(branch)
3195                             parent = self.knownBranches[branch]
3196                             if parent == branch:
3197                                 parent = ""
3198                             else:
3199                                 fullBranch = self.projectName + branch
3200                                 if fullBranch not in self.p4BranchesInGit:
3201                                     if not self.silent:
3202                                         print("\n    Importing new branch %s" % fullBranch);
3203                                     if self.importNewBranch(branch, change - 1):
3204                                         parent = ""
3205                                         self.p4BranchesInGit.append(fullBranch)
3206                                     if not self.silent:
3207                                         print("\n    Resuming with change %s" % change);
3208
3209                                 if self.verbose:
3210                                     print "parent determined through known branches: %s" % parent
3211
3212                         branch = self.gitRefForBranch(branch)
3213                         parent = self.gitRefForBranch(parent)
3214
3215                         if self.verbose:
3216                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3217
3218                         if len(parent) == 0 and branch in self.initialParents:
3219                             parent = self.initialParents[branch]
3220                             del self.initialParents[branch]
3221
3222                         blob = None
3223                         if len(parent) > 0:
3224                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3225                             if self.verbose:
3226                                 print "Creating temporary branch: " + tempBranch
3227                             self.commit(description, filesForCommit, tempBranch)
3228                             self.tempBranches.append(tempBranch)
3229                             self.checkpoint()
3230                             blob = self.searchParent(parent, branch, tempBranch)
3231                         if blob:
3232                             self.commit(description, filesForCommit, branch, blob)
3233                         else:
3234                             if self.verbose:
3235                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3236                             self.commit(description, filesForCommit, branch, parent)
3237                 else:
3238                     files = self.extractFilesFromCommit(description)
3239                     self.commit(description, files, self.branch,
3240                                 self.initialParent)
3241                     # only needed once, to connect to the previous commit
3242                     self.initialParent = ""
3243             except IOError:
3244                 print self.gitError.read()
3245                 sys.exit(1)
3246
3247     def importHeadRevision(self, revision):
3248         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3249
3250         details = {}
3251         details["user"] = "git perforce import user"
3252         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3253                            % (' '.join(self.depotPaths), revision))
3254         details["change"] = revision
3255         newestRevision = 0
3256
3257         fileCnt = 0
3258         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3259
3260         for info in p4CmdList(["files"] + fileArgs):
3261
3262             if 'code' in info and info['code'] == 'error':
3263                 sys.stderr.write("p4 returned an error: %s\n"
3264                                  % info['data'])
3265                 if info['data'].find("must refer to client") >= 0:
3266                     sys.stderr.write("This particular p4 error is misleading.\n")
3267                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3268                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3269                 sys.exit(1)
3270             if 'p4ExitCode' in info:
3271                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3272                 sys.exit(1)
3273
3274
3275             change = int(info["change"])
3276             if change > newestRevision:
3277                 newestRevision = change
3278
3279             if info["action"] in self.delete_actions:
3280                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3281                 #fileCnt = fileCnt + 1
3282                 continue
3283
3284             for prop in ["depotFile", "rev", "action", "type" ]:
3285                 details["%s%s" % (prop, fileCnt)] = info[prop]
3286
3287             fileCnt = fileCnt + 1
3288
3289         details["change"] = newestRevision
3290
3291         # Use time from top-most change so that all git p4 clones of
3292         # the same p4 repo have the same commit SHA1s.
3293         res = p4_describe(newestRevision)
3294         details["time"] = res["time"]
3295
3296         self.updateOptionDict(details)
3297         try:
3298             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3299         except IOError:
3300             print "IO error with git fast-import. Is your git version recent enough?"
3301             print self.gitError.read()
3302
3303
3304     def run(self, args):
3305         self.depotPaths = []
3306         self.changeRange = ""
3307         self.previousDepotPaths = []
3308         self.hasOrigin = False
3309
3310         # map from branch depot path to parent branch
3311         self.knownBranches = {}
3312         self.initialParents = {}
3313
3314         if self.importIntoRemotes:
3315             self.refPrefix = "refs/remotes/p4/"
3316         else:
3317             self.refPrefix = "refs/heads/p4/"
3318
3319         if self.syncWithOrigin:
3320             self.hasOrigin = originP4BranchesExist()
3321             if self.hasOrigin:
3322                 if not self.silent:
3323                     print 'Syncing with origin first, using "git fetch origin"'
3324                 system("git fetch origin")
3325
3326         branch_arg_given = bool(self.branch)
3327         if len(self.branch) == 0:
3328             self.branch = self.refPrefix + "master"
3329             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3330                 system("git update-ref %s refs/heads/p4" % self.branch)
3331                 system("git branch -D p4")
3332
3333         # accept either the command-line option, or the configuration variable
3334         if self.useClientSpec:
3335             # will use this after clone to set the variable
3336             self.useClientSpec_from_options = True
3337         else:
3338             if gitConfigBool("git-p4.useclientspec"):
3339                 self.useClientSpec = True
3340         if self.useClientSpec:
3341             self.clientSpecDirs = getClientSpec()
3342
3343         # TODO: should always look at previous commits,
3344         # merge with previous imports, if possible.
3345         if args == []:
3346             if self.hasOrigin:
3347                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3348
3349             # branches holds mapping from branch name to sha1
3350             branches = p4BranchesInGit(self.importIntoRemotes)
3351
3352             # restrict to just this one, disabling detect-branches
3353             if branch_arg_given:
3354                 short = self.branch.split("/")[-1]
3355                 if short in branches:
3356                     self.p4BranchesInGit = [ short ]
3357             else:
3358                 self.p4BranchesInGit = branches.keys()
3359
3360             if len(self.p4BranchesInGit) > 1:
3361                 if not self.silent:
3362                     print "Importing from/into multiple branches"
3363                 self.detectBranches = True
3364                 for branch in branches.keys():
3365                     self.initialParents[self.refPrefix + branch] = \
3366                         branches[branch]
3367
3368             if self.verbose:
3369                 print "branches: %s" % self.p4BranchesInGit
3370
3371             p4Change = 0
3372             for branch in self.p4BranchesInGit:
3373                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3374
3375                 settings = extractSettingsGitLog(logMsg)
3376
3377                 self.readOptions(settings)
3378                 if (settings.has_key('depot-paths')
3379                     and settings.has_key ('change')):
3380                     change = int(settings['change']) + 1
3381                     p4Change = max(p4Change, change)
3382
3383                     depotPaths = sorted(settings['depot-paths'])
3384                     if self.previousDepotPaths == []:
3385                         self.previousDepotPaths = depotPaths
3386                     else:
3387                         paths = []
3388                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3389                             prev_list = prev.split("/")
3390                             cur_list = cur.split("/")
3391                             for i in range(0, min(len(cur_list), len(prev_list))):
3392                                 if cur_list[i] <> prev_list[i]:
3393                                     i = i - 1
3394                                     break
3395
3396                             paths.append ("/".join(cur_list[:i + 1]))
3397
3398                         self.previousDepotPaths = paths
3399
3400             if p4Change > 0:
3401                 self.depotPaths = sorted(self.previousDepotPaths)
3402                 self.changeRange = "@%s,#head" % p4Change
3403                 if not self.silent and not self.detectBranches:
3404                     print "Performing incremental import into %s git branch" % self.branch
3405
3406         # accept multiple ref name abbreviations:
3407         #    refs/foo/bar/branch -> use it exactly
3408         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3409         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3410         if not self.branch.startswith("refs/"):
3411             if self.importIntoRemotes:
3412                 prepend = "refs/remotes/"
3413             else:
3414                 prepend = "refs/heads/"
3415             if not self.branch.startswith("p4/"):
3416                 prepend += "p4/"
3417             self.branch = prepend + self.branch
3418
3419         if len(args) == 0 and self.depotPaths:
3420             if not self.silent:
3421                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3422         else:
3423             if self.depotPaths and self.depotPaths != args:
3424                 print ("previous import used depot path %s and now %s was specified. "
3425                        "This doesn't work!" % (' '.join (self.depotPaths),
3426                                                ' '.join (args)))
3427                 sys.exit(1)
3428
3429             self.depotPaths = sorted(args)
3430
3431         revision = ""
3432         self.users = {}
3433
3434         # Make sure no revision specifiers are used when --changesfile
3435         # is specified.
3436         bad_changesfile = False
3437         if len(self.changesFile) > 0:
3438             for p in self.depotPaths:
3439                 if p.find("@") >= 0 or p.find("#") >= 0:
3440                     bad_changesfile = True
3441                     break
3442         if bad_changesfile:
3443             die("Option --changesfile is incompatible with revision specifiers")
3444
3445         newPaths = []
3446         for p in self.depotPaths:
3447             if p.find("@") != -1:
3448                 atIdx = p.index("@")
3449                 self.changeRange = p[atIdx:]
3450                 if self.changeRange == "@all":
3451                     self.changeRange = ""
3452                 elif ',' not in self.changeRange:
3453                     revision = self.changeRange
3454                     self.changeRange = ""
3455                 p = p[:atIdx]
3456             elif p.find("#") != -1:
3457                 hashIdx = p.index("#")
3458                 revision = p[hashIdx:]
3459                 p = p[:hashIdx]
3460             elif self.previousDepotPaths == []:
3461                 # pay attention to changesfile, if given, else import
3462                 # the entire p4 tree at the head revision
3463                 if len(self.changesFile) == 0:
3464                     revision = "#head"
3465
3466             p = re.sub ("\.\.\.$", "", p)
3467             if not p.endswith("/"):
3468                 p += "/"
3469
3470             newPaths.append(p)
3471
3472         self.depotPaths = newPaths
3473
3474         # --detect-branches may change this for each branch
3475         self.branchPrefixes = self.depotPaths
3476
3477         self.loadUserMapFromCache()
3478         self.labels = {}
3479         if self.detectLabels:
3480             self.getLabels();
3481
3482         if self.detectBranches:
3483             ## FIXME - what's a P4 projectName ?
3484             self.projectName = self.guessProjectName()
3485
3486             if self.hasOrigin:
3487                 self.getBranchMappingFromGitBranches()
3488             else:
3489                 self.getBranchMapping()
3490             if self.verbose:
3491                 print "p4-git branches: %s" % self.p4BranchesInGit
3492                 print "initial parents: %s" % self.initialParents
3493             for b in self.p4BranchesInGit:
3494                 if b != "master":
3495
3496                     ## FIXME
3497                     b = b[len(self.projectName):]
3498                 self.createdBranches.add(b)
3499
3500         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3501
3502         self.importProcess = subprocess.Popen(["git", "fast-import"],
3503                                               stdin=subprocess.PIPE,
3504                                               stdout=subprocess.PIPE,
3505                                               stderr=subprocess.PIPE);
3506         self.gitOutput = self.importProcess.stdout
3507         self.gitStream = self.importProcess.stdin
3508         self.gitError = self.importProcess.stderr
3509
3510         if revision:
3511             self.importHeadRevision(revision)
3512         else:
3513             changes = []
3514
3515             if len(self.changesFile) > 0:
3516                 output = open(self.changesFile).readlines()
3517                 changeSet = set()
3518                 for line in output:
3519                     changeSet.add(int(line))
3520
3521                 for change in changeSet:
3522                     changes.append(change)
3523
3524                 changes.sort()
3525             else:
3526                 # catch "git p4 sync" with no new branches, in a repo that
3527                 # does not have any existing p4 branches
3528                 if len(args) == 0:
3529                     if not self.p4BranchesInGit:
3530                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3531
3532                     # The default branch is master, unless --branch is used to
3533                     # specify something else.  Make sure it exists, or complain
3534                     # nicely about how to use --branch.
3535                     if not self.detectBranches:
3536                         if not branch_exists(self.branch):
3537                             if branch_arg_given:
3538                                 die("Error: branch %s does not exist." % self.branch)
3539                             else:
3540                                 die("Error: no branch %s; perhaps specify one with --branch." %
3541                                     self.branch)
3542
3543                 if self.verbose:
3544                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3545                                                               self.changeRange)
3546                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3547
3548                 if len(self.maxChanges) > 0:
3549                     changes = changes[:min(int(self.maxChanges), len(changes))]
3550
3551             if len(changes) == 0:
3552                 if not self.silent:
3553                     print "No changes to import!"
3554             else:
3555                 if not self.silent and not self.detectBranches:
3556                     print "Import destination: %s" % self.branch
3557
3558                 self.updatedBranches = set()
3559
3560                 if not self.detectBranches:
3561                     if args:
3562                         # start a new branch
3563                         self.initialParent = ""
3564                     else:
3565                         # build on a previous revision
3566                         self.initialParent = parseRevision(self.branch)
3567
3568                 self.importChanges(changes)
3569
3570                 if not self.silent:
3571                     print ""
3572                     if len(self.updatedBranches) > 0:
3573                         sys.stdout.write("Updated branches: ")
3574                         for b in self.updatedBranches:
3575                             sys.stdout.write("%s " % b)
3576                         sys.stdout.write("\n")
3577
3578         if gitConfigBool("git-p4.importLabels"):
3579             self.importLabels = True
3580
3581         if self.importLabels:
3582             p4Labels = getP4Labels(self.depotPaths)
3583             gitTags = getGitTags()
3584
3585             missingP4Labels = p4Labels - gitTags
3586             self.importP4Labels(self.gitStream, missingP4Labels)
3587
3588         self.gitStream.close()
3589         if self.importProcess.wait() != 0:
3590             die("fast-import failed: %s" % self.gitError.read())
3591         self.gitOutput.close()
3592         self.gitError.close()
3593
3594         # Cleanup temporary branches created during import
3595         if self.tempBranches != []:
3596             for branch in self.tempBranches:
3597                 read_pipe("git update-ref -d %s" % branch)
3598             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3599
3600         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3601         # a convenient shortcut refname "p4".
3602         if self.importIntoRemotes:
3603             head_ref = self.refPrefix + "HEAD"
3604             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3605                 system(["git", "symbolic-ref", head_ref, self.branch])
3606
3607         return True
3608
3609 class P4Rebase(Command):
3610     def __init__(self):
3611         Command.__init__(self)
3612         self.options = [
3613                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3614         ]
3615         self.importLabels = False
3616         self.description = ("Fetches the latest revision from perforce and "
3617                             + "rebases the current work (branch) against it")
3618
3619     def run(self, args):
3620         sync = P4Sync()
3621         sync.importLabels = self.importLabels
3622         sync.run([])
3623
3624         return self.rebase()
3625
3626     def rebase(self):
3627         if os.system("git update-index --refresh") != 0:
3628             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3629         if len(read_pipe("git diff-index HEAD --")) > 0:
3630             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3631
3632         [upstream, settings] = findUpstreamBranchPoint()
3633         if len(upstream) == 0:
3634             die("Cannot find upstream branchpoint for rebase")
3635
3636         # the branchpoint may be p4/foo~3, so strip off the parent
3637         upstream = re.sub("~[0-9]+$", "", upstream)
3638
3639         print "Rebasing the current branch onto %s" % upstream
3640         oldHead = read_pipe("git rev-parse HEAD").strip()
3641         system("git rebase %s" % upstream)
3642         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3643         return True
3644
3645 class P4Clone(P4Sync):
3646     def __init__(self):
3647         P4Sync.__init__(self)
3648         self.description = "Creates a new git repository and imports from Perforce into it"
3649         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3650         self.options += [
3651             optparse.make_option("--destination", dest="cloneDestination",
3652                                  action='store', default=None,
3653                                  help="where to leave result of the clone"),
3654             optparse.make_option("--bare", dest="cloneBare",
3655                                  action="store_true", default=False),
3656         ]
3657         self.cloneDestination = None
3658         self.needsGit = False
3659         self.cloneBare = False
3660
3661     def defaultDestination(self, args):
3662         ## TODO: use common prefix of args?
3663         depotPath = args[0]
3664         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3665         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3666         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3667         depotDir = re.sub(r"/$", "", depotDir)
3668         return os.path.split(depotDir)[1]
3669
3670     def run(self, args):
3671         if len(args) < 1:
3672             return False
3673
3674         if self.keepRepoPath and not self.cloneDestination:
3675             sys.stderr.write("Must specify destination for --keep-path\n")
3676             sys.exit(1)
3677
3678         depotPaths = args
3679
3680         if not self.cloneDestination and len(depotPaths) > 1:
3681             self.cloneDestination = depotPaths[-1]
3682             depotPaths = depotPaths[:-1]
3683
3684         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3685         for p in depotPaths:
3686             if not p.startswith("//"):
3687                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3688                 return False
3689
3690         if not self.cloneDestination:
3691             self.cloneDestination = self.defaultDestination(args)
3692
3693         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3694
3695         if not os.path.exists(self.cloneDestination):
3696             os.makedirs(self.cloneDestination)
3697         chdir(self.cloneDestination)
3698
3699         init_cmd = [ "git", "init" ]
3700         if self.cloneBare:
3701             init_cmd.append("--bare")
3702         retcode = subprocess.call(init_cmd)
3703         if retcode:
3704             raise CalledProcessError(retcode, init_cmd)
3705
3706         if not P4Sync.run(self, depotPaths):
3707             return False
3708
3709         # create a master branch and check out a work tree
3710         if gitBranchExists(self.branch):
3711             system([ "git", "branch", "master", self.branch ])
3712             if not self.cloneBare:
3713                 system([ "git", "checkout", "-f" ])
3714         else:
3715             print 'Not checking out any branch, use ' \
3716                   '"git checkout -q -b master <branch>"'
3717
3718         # auto-set this variable if invoked with --use-client-spec
3719         if self.useClientSpec_from_options:
3720             system("git config --bool git-p4.useclientspec true")
3721
3722         return True
3723
3724 class P4Branches(Command):
3725     def __init__(self):
3726         Command.__init__(self)
3727         self.options = [ ]
3728         self.description = ("Shows the git branches that hold imports and their "
3729                             + "corresponding perforce depot paths")
3730         self.verbose = False
3731
3732     def run(self, args):
3733         if originP4BranchesExist():
3734             createOrUpdateBranchesFromOrigin()
3735
3736         cmdline = "git rev-parse --symbolic "
3737         cmdline += " --remotes"
3738
3739         for line in read_pipe_lines(cmdline):
3740             line = line.strip()
3741
3742             if not line.startswith('p4/') or line == "p4/HEAD":
3743                 continue
3744             branch = line
3745
3746             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3747             settings = extractSettingsGitLog(log)
3748
3749             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3750         return True
3751
3752 class HelpFormatter(optparse.IndentedHelpFormatter):
3753     def __init__(self):
3754         optparse.IndentedHelpFormatter.__init__(self)
3755
3756     def format_description(self, description):
3757         if description:
3758             return description + "\n"
3759         else:
3760             return ""
3761
3762 def printUsage(commands):
3763     print "usage: %s <command> [options]" % sys.argv[0]
3764     print ""
3765     print "valid commands: %s" % ", ".join(commands)
3766     print ""
3767     print "Try %s <command> --help for command specific help." % sys.argv[0]
3768     print ""
3769
3770 commands = {
3771     "debug" : P4Debug,
3772     "submit" : P4Submit,
3773     "commit" : P4Submit,
3774     "sync" : P4Sync,
3775     "rebase" : P4Rebase,
3776     "clone" : P4Clone,
3777     "rollback" : P4RollBack,
3778     "branches" : P4Branches
3779 }
3780
3781
3782 def main():
3783     if len(sys.argv[1:]) == 0:
3784         printUsage(commands.keys())
3785         sys.exit(2)
3786
3787     cmdName = sys.argv[1]
3788     try:
3789         klass = commands[cmdName]
3790         cmd = klass()
3791     except KeyError:
3792         print "unknown command %s" % cmdName
3793         print ""
3794         printUsage(commands.keys())
3795         sys.exit(2)
3796
3797     options = cmd.options
3798     cmd.gitdir = os.environ.get("GIT_DIR", None)
3799
3800     args = sys.argv[2:]
3801
3802     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3803     if cmd.needsGit:
3804         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3805
3806     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3807                                    options,
3808                                    description = cmd.description,
3809                                    formatter = HelpFormatter())
3810
3811     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3812     global verbose
3813     verbose = cmd.verbose
3814     if cmd.needsGit:
3815         if cmd.gitdir == None:
3816             cmd.gitdir = os.path.abspath(".git")
3817             if not isValidGitDir(cmd.gitdir):
3818                 # "rev-parse --git-dir" without arguments will try $PWD/.git
3819                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3820                 if os.path.exists(cmd.gitdir):
3821                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3822                     if len(cdup) > 0:
3823                         chdir(cdup);
3824
3825         if not isValidGitDir(cmd.gitdir):
3826             if isValidGitDir(cmd.gitdir + "/.git"):
3827                 cmd.gitdir += "/.git"
3828             else:
3829                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3830
3831         # so git commands invoked from the P4 workspace will succeed
3832         os.environ["GIT_DIR"] = cmd.gitdir
3833
3834     if not cmd.run(args):
3835         parser.print_help()
3836         sys.exit(2)
3837
3838
3839 if __name__ == '__main__':
3840     main()