Merge branch 'nd/files-backend-git-dir' into next
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31     from subprocess import CalledProcessError
32 except ImportError:
33     # from python2.7:subprocess.py
34     # Exception classes used by this module.
35     class CalledProcessError(Exception):
36         """This exception is raised when a process run by check_call() returns
37         a non-zero exit status.  The exit status will be stored in the
38         returncode attribute."""
39         def __init__(self, returncode, cmd):
40             self.returncode = returncode
41             self.cmd = cmd
42         def __str__(self):
43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54     """Build a suitable p4 command line.
55
56     This consolidates building and returning a p4 command line into one
57     location. It means that hooking into the environment, or other configuration
58     can be done more easily.
59     """
60     real_cmd = ["p4"]
61
62     user = gitConfig("git-p4.user")
63     if len(user) > 0:
64         real_cmd += ["-u",user]
65
66     password = gitConfig("git-p4.password")
67     if len(password) > 0:
68         real_cmd += ["-P", password]
69
70     port = gitConfig("git-p4.port")
71     if len(port) > 0:
72         real_cmd += ["-p", port]
73
74     host = gitConfig("git-p4.host")
75     if len(host) > 0:
76         real_cmd += ["-H", host]
77
78     client = gitConfig("git-p4.client")
79     if len(client) > 0:
80         real_cmd += ["-c", client]
81
82     retries = gitConfigInt("git-p4.retries")
83     if retries is None:
84         # Perform 3 retries by default
85         retries = 3
86     if retries > 0:
87         # Provide a way to not pass this option by setting git-p4.retries to 0
88         real_cmd += ["-r", str(retries)]
89
90     if isinstance(cmd,basestring):
91         real_cmd = ' '.join(real_cmd) + ' ' + cmd
92     else:
93         real_cmd += cmd
94     return real_cmd
95
96 def git_dir(path):
97     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98         This won't automatically add ".git" to a directory.
99     """
100     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101     if not d or len(d) == 0:
102         return None
103     else:
104         return d
105
106 def chdir(path, is_client_path=False):
107     """Do chdir to the given path, and set the PWD environment
108        variable for use by P4.  It does not look at getcwd() output.
109        Since we're not using the shell, it is necessary to set the
110        PWD environment variable explicitly.
111
112        Normally, expand the path to force it to be absolute.  This
113        addresses the use of relative path names inside P4 settings,
114        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
115        as given; it looks for .p4config using PWD.
116
117        If is_client_path, the path was handed to us directly by p4,
118        and may be a symbolic link.  Do not call os.getcwd() in this
119        case, because it will cause p4 to think that PWD is not inside
120        the client path.
121        """
122
123     os.chdir(path)
124     if not is_client_path:
125         path = os.getcwd()
126     os.environ['PWD'] = path
127
128 def calcDiskFree():
129     """Return free space in bytes on the disk of the given dirname."""
130     if platform.system() == 'Windows':
131         free_bytes = ctypes.c_ulonglong(0)
132         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133         return free_bytes.value
134     else:
135         st = os.statvfs(os.getcwd())
136         return st.f_bavail * st.f_frsize
137
138 def die(msg):
139     if verbose:
140         raise Exception(msg)
141     else:
142         sys.stderr.write(msg + "\n")
143         sys.exit(1)
144
145 def write_pipe(c, stdin):
146     if verbose:
147         sys.stderr.write('Writing pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151     pipe = p.stdin
152     val = pipe.write(stdin)
153     pipe.close()
154     if p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_write_pipe(c, stdin):
160     real_cmd = p4_build_cmd(c)
161     return write_pipe(real_cmd, stdin)
162
163 def read_pipe_full(c):
164     """ Read output from  command. Returns a tuple
165         of the return status, stdout text and stderr
166         text.
167     """
168     if verbose:
169         sys.stderr.write('Reading pipe: %s\n' % str(c))
170
171     expand = isinstance(c,basestring)
172     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173     (out, err) = p.communicate()
174     return (p.returncode, out, err)
175
176 def read_pipe(c, ignore_error=False):
177     """ Read output from  command. Returns the output text on
178         success. On failure, terminates execution, unless
179         ignore_error is True, when it returns an empty string.
180     """
181     (retcode, out, err) = read_pipe_full(c)
182     if retcode != 0:
183         if ignore_error:
184             out = ""
185         else:
186             die('Command failed: %s\nError: %s' % (str(c), err))
187     return out
188
189 def read_pipe_text(c):
190     """ Read output from a command with trailing whitespace stripped.
191         On error, returns None.
192     """
193     (retcode, out, err) = read_pipe_full(c)
194     if retcode != 0:
195         return None
196     else:
197         return out.rstrip()
198
199 def p4_read_pipe(c, ignore_error=False):
200     real_cmd = p4_build_cmd(c)
201     return read_pipe(real_cmd, ignore_error)
202
203 def read_pipe_lines(c):
204     if verbose:
205         sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207     expand = isinstance(c, basestring)
208     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209     pipe = p.stdout
210     val = pipe.readlines()
211     if pipe.close() or p.wait():
212         die('Command failed: %s' % str(c))
213
214     return val
215
216 def p4_read_pipe_lines(c):
217     """Specifically invoke p4 on the command supplied. """
218     real_cmd = p4_build_cmd(c)
219     return read_pipe_lines(real_cmd)
220
221 def p4_has_command(cmd):
222     """Ask p4 for help on this command.  If it returns an error, the
223        command does not exist in this version of p4."""
224     real_cmd = p4_build_cmd(["help", cmd])
225     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226                                    stderr=subprocess.PIPE)
227     p.communicate()
228     return p.returncode == 0
229
230 def p4_has_move_command():
231     """See if the move command exists, that it supports -k, and that
232        it has not been administratively disabled.  The arguments
233        must be correct, but the filenames do not have to exist.  Use
234        ones with wildcards so even if they exist, it will fail."""
235
236     if not p4_has_command("move"):
237         return False
238     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240     (out, err) = p.communicate()
241     # return code will be 1 in either case
242     if err.find("Invalid option") >= 0:
243         return False
244     if err.find("disabled") >= 0:
245         return False
246     # assume it failed because @... was invalid changelist
247     return True
248
249 def system(cmd, ignore_error=False):
250     expand = isinstance(cmd,basestring)
251     if verbose:
252         sys.stderr.write("executing %s\n" % str(cmd))
253     retcode = subprocess.call(cmd, shell=expand)
254     if retcode and not ignore_error:
255         raise CalledProcessError(retcode, cmd)
256
257     return retcode
258
259 def p4_system(cmd):
260     """Specifically invoke p4 as the system command. """
261     real_cmd = p4_build_cmd(cmd)
262     expand = isinstance(real_cmd, basestring)
263     retcode = subprocess.call(real_cmd, shell=expand)
264     if retcode:
265         raise CalledProcessError(retcode, real_cmd)
266
267 _p4_version_string = None
268 def p4_version_string():
269     """Read the version string, showing just the last line, which
270        hopefully is the interesting version bit.
271
272        $ p4 -V
273        Perforce - The Fast Software Configuration Management System.
274        Copyright 1995-2011 Perforce Software.  All rights reserved.
275        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276     """
277     global _p4_version_string
278     if not _p4_version_string:
279         a = p4_read_pipe_lines(["-V"])
280         _p4_version_string = a[-1].rstrip()
281     return _p4_version_string
282
283 def p4_integrate(src, dest):
284     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
285
286 def p4_sync(f, *options):
287     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
288
289 def p4_add(f):
290     # forcibly add file names with wildcards
291     if wildcard_present(f):
292         p4_system(["add", "-f", f])
293     else:
294         p4_system(["add", f])
295
296 def p4_delete(f):
297     p4_system(["delete", wildcard_encode(f)])
298
299 def p4_edit(f, *options):
300     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
301
302 def p4_revert(f):
303     p4_system(["revert", wildcard_encode(f)])
304
305 def p4_reopen(type, f):
306     p4_system(["reopen", "-t", type, wildcard_encode(f)])
307
308 def p4_reopen_in_change(changelist, files):
309     cmd = ["reopen", "-c", str(changelist)] + files
310     p4_system(cmd)
311
312 def p4_move(src, dest):
313     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
315 def p4_last_change():
316     results = p4CmdList(["changes", "-m", "1"])
317     return int(results[0]['change'])
318
319 def p4_describe(change):
320     """Make sure it returns a valid result by checking for
321        the presence of field "time".  Return a dict of the
322        results."""
323
324     ds = p4CmdList(["describe", "-s", str(change)])
325     if len(ds) != 1:
326         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328     d = ds[0]
329
330     if "p4ExitCode" in d:
331         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332                                                       str(d)))
333     if "code" in d:
334         if d["code"] == "error":
335             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337     if "time" not in d:
338         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340     return d
341
342 #
343 # Canonicalize the p4 type and return a tuple of the
344 # base type, plus any modifiers.  See "p4 help filetypes"
345 # for a list and explanation.
346 #
347 def split_p4_type(p4type):
348
349     p4_filetypes_historical = {
350         "ctempobj": "binary+Sw",
351         "ctext": "text+C",
352         "cxtext": "text+Cx",
353         "ktext": "text+k",
354         "kxtext": "text+kx",
355         "ltext": "text+F",
356         "tempobj": "binary+FSw",
357         "ubinary": "binary+F",
358         "uresource": "resource+F",
359         "uxbinary": "binary+Fx",
360         "xbinary": "binary+x",
361         "xltext": "text+Fx",
362         "xtempobj": "binary+Swx",
363         "xtext": "text+x",
364         "xunicode": "unicode+x",
365         "xutf16": "utf16+x",
366     }
367     if p4type in p4_filetypes_historical:
368         p4type = p4_filetypes_historical[p4type]
369     mods = ""
370     s = p4type.split("+")
371     base = s[0]
372     mods = ""
373     if len(s) > 1:
374         mods = s[1]
375     return (base, mods)
376
377 #
378 # return the raw p4 type of a file (text, text+ko, etc)
379 #
380 def p4_type(f):
381     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
382     return results[0]['headType']
383
384 #
385 # Given a type base and modifier, return a regexp matching
386 # the keywords that can be expanded in the file
387 #
388 def p4_keywords_regexp_for_type(base, type_mods):
389     if base in ("text", "unicode", "binary"):
390         kwords = None
391         if "ko" in type_mods:
392             kwords = 'Id|Header'
393         elif "k" in type_mods:
394             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395         else:
396             return None
397         pattern = r"""
398             \$              # Starts with a dollar, followed by...
399             (%s)            # one of the keywords, followed by...
400             (:[^$\n]+)?     # possibly an old expansion, followed by...
401             \$              # another dollar
402             """ % kwords
403         return pattern
404     else:
405         return None
406
407 #
408 # Given a file, return a regexp matching the possible
409 # RCS keywords that will be expanded, or None for files
410 # with kw expansion turned off.
411 #
412 def p4_keywords_regexp_for_file(file):
413     if not os.path.exists(file):
414         return None
415     else:
416         (type_base, type_mods) = split_p4_type(p4_type(file))
417         return p4_keywords_regexp_for_type(type_base, type_mods)
418
419 def setP4ExecBit(file, mode):
420     # Reopens an already open file and changes the execute bit to match
421     # the execute bit setting in the passed in mode.
422
423     p4Type = "+x"
424
425     if not isModeExec(mode):
426         p4Type = getP4OpenedType(file)
427         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429         if p4Type[-1] == "+":
430             p4Type = p4Type[0:-1]
431
432     p4_reopen(p4Type, file)
433
434 def getP4OpenedType(file):
435     # Returns the perforce file type for the given file.
436
437     result = p4_read_pipe(["opened", wildcard_encode(file)])
438     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
439     if match:
440         return match.group(1)
441     else:
442         die("Could not determine file type for %s (result: '%s')" % (file, result))
443
444 # Return the set of all p4 labels
445 def getP4Labels(depotPaths):
446     labels = set()
447     if isinstance(depotPaths,basestring):
448         depotPaths = [depotPaths]
449
450     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451         label = l['label']
452         labels.add(label)
453
454     return labels
455
456 # Return the set of all git tags
457 def getGitTags():
458     gitTags = set()
459     for line in read_pipe_lines(["git", "tag"]):
460         tag = line.strip()
461         gitTags.add(tag)
462     return gitTags
463
464 def diffTreePattern():
465     # This is a simple generator for the diff tree regex pattern. This could be
466     # a class variable if this and parseDiffTreeEntry were a part of a class.
467     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468     while True:
469         yield pattern
470
471 def parseDiffTreeEntry(entry):
472     """Parses a single diff tree entry into its component elements.
473
474     See git-diff-tree(1) manpage for details about the format of the diff
475     output. This method returns a dictionary with the following elements:
476
477     src_mode - The mode of the source file
478     dst_mode - The mode of the destination file
479     src_sha1 - The sha1 for the source file
480     dst_sha1 - The sha1 fr the destination file
481     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482     status_score - The score for the status (applicable for 'C' and 'R'
483                    statuses). This is None if there is no score.
484     src - The path for the source file.
485     dst - The path for the destination file. This is only present for
486           copy or renames. If it is not present, this is None.
487
488     If the pattern is not matched, None is returned."""
489
490     match = diffTreePattern().next().match(entry)
491     if match:
492         return {
493             'src_mode': match.group(1),
494             'dst_mode': match.group(2),
495             'src_sha1': match.group(3),
496             'dst_sha1': match.group(4),
497             'status': match.group(5),
498             'status_score': match.group(6),
499             'src': match.group(7),
500             'dst': match.group(10)
501         }
502     return None
503
504 def isModeExec(mode):
505     # Returns True if the given git mode represents an executable file,
506     # otherwise False.
507     return mode[-3:] == "755"
508
509 def isModeExecChanged(src_mode, dst_mode):
510     return isModeExec(src_mode) != isModeExec(dst_mode)
511
512 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
513
514     if isinstance(cmd,basestring):
515         cmd = "-G " + cmd
516         expand = True
517     else:
518         cmd = ["-G"] + cmd
519         expand = False
520
521     cmd = p4_build_cmd(cmd)
522     if verbose:
523         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
524
525     # Use a temporary file to avoid deadlocks without
526     # subprocess.communicate(), which would put another copy
527     # of stdout into memory.
528     stdin_file = None
529     if stdin is not None:
530         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
531         if isinstance(stdin,basestring):
532             stdin_file.write(stdin)
533         else:
534             for i in stdin:
535                 stdin_file.write(i + '\n')
536         stdin_file.flush()
537         stdin_file.seek(0)
538
539     p4 = subprocess.Popen(cmd,
540                           shell=expand,
541                           stdin=stdin_file,
542                           stdout=subprocess.PIPE)
543
544     result = []
545     try:
546         while True:
547             entry = marshal.load(p4.stdout)
548             if cb is not None:
549                 cb(entry)
550             else:
551                 result.append(entry)
552     except EOFError:
553         pass
554     exitCode = p4.wait()
555     if exitCode != 0:
556         entry = {}
557         entry["p4ExitCode"] = exitCode
558         result.append(entry)
559
560     return result
561
562 def p4Cmd(cmd):
563     list = p4CmdList(cmd)
564     result = {}
565     for entry in list:
566         result.update(entry)
567     return result;
568
569 def p4Where(depotPath):
570     if not depotPath.endswith("/"):
571         depotPath += "/"
572     depotPathLong = depotPath + "..."
573     outputList = p4CmdList(["where", depotPathLong])
574     output = None
575     for entry in outputList:
576         if "depotFile" in entry:
577             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
578             # The base path always ends with "/...".
579             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
580                 output = entry
581                 break
582         elif "data" in entry:
583             data = entry.get("data")
584             space = data.find(" ")
585             if data[:space] == depotPath:
586                 output = entry
587                 break
588     if output == None:
589         return ""
590     if output["code"] == "error":
591         return ""
592     clientPath = ""
593     if "path" in output:
594         clientPath = output.get("path")
595     elif "data" in output:
596         data = output.get("data")
597         lastSpace = data.rfind(" ")
598         clientPath = data[lastSpace + 1:]
599
600     if clientPath.endswith("..."):
601         clientPath = clientPath[:-3]
602     return clientPath
603
604 def currentGitBranch():
605     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
606
607 def isValidGitDir(path):
608     return git_dir(path) != None
609
610 def parseRevision(ref):
611     return read_pipe("git rev-parse %s" % ref).strip()
612
613 def branchExists(ref):
614     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
615                      ignore_error=True)
616     return len(rev) > 0
617
618 def extractLogMessageFromGitCommit(commit):
619     logMessage = ""
620
621     ## fixme: title is first line of commit, not 1st paragraph.
622     foundTitle = False
623     for log in read_pipe_lines("git cat-file commit %s" % commit):
624        if not foundTitle:
625            if len(log) == 1:
626                foundTitle = True
627            continue
628
629        logMessage += log
630     return logMessage
631
632 def extractSettingsGitLog(log):
633     values = {}
634     for line in log.split("\n"):
635         line = line.strip()
636         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
637         if not m:
638             continue
639
640         assignments = m.group(1).split (':')
641         for a in assignments:
642             vals = a.split ('=')
643             key = vals[0].strip()
644             val = ('='.join (vals[1:])).strip()
645             if val.endswith ('\"') and val.startswith('"'):
646                 val = val[1:-1]
647
648             values[key] = val
649
650     paths = values.get("depot-paths")
651     if not paths:
652         paths = values.get("depot-path")
653     if paths:
654         values['depot-paths'] = paths.split(',')
655     return values
656
657 def gitBranchExists(branch):
658     proc = subprocess.Popen(["git", "rev-parse", branch],
659                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
660     return proc.wait() == 0;
661
662 _gitConfig = {}
663
664 def gitConfig(key, typeSpecifier=None):
665     if not _gitConfig.has_key(key):
666         cmd = [ "git", "config" ]
667         if typeSpecifier:
668             cmd += [ typeSpecifier ]
669         cmd += [ key ]
670         s = read_pipe(cmd, ignore_error=True)
671         _gitConfig[key] = s.strip()
672     return _gitConfig[key]
673
674 def gitConfigBool(key):
675     """Return a bool, using git config --bool.  It is True only if the
676        variable is set to true, and False if set to false or not present
677        in the config."""
678
679     if not _gitConfig.has_key(key):
680         _gitConfig[key] = gitConfig(key, '--bool') == "true"
681     return _gitConfig[key]
682
683 def gitConfigInt(key):
684     if not _gitConfig.has_key(key):
685         cmd = [ "git", "config", "--int", key ]
686         s = read_pipe(cmd, ignore_error=True)
687         v = s.strip()
688         try:
689             _gitConfig[key] = int(gitConfig(key, '--int'))
690         except ValueError:
691             _gitConfig[key] = None
692     return _gitConfig[key]
693
694 def gitConfigList(key):
695     if not _gitConfig.has_key(key):
696         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
697         _gitConfig[key] = s.strip().splitlines()
698         if _gitConfig[key] == ['']:
699             _gitConfig[key] = []
700     return _gitConfig[key]
701
702 def p4BranchesInGit(branchesAreInRemotes=True):
703     """Find all the branches whose names start with "p4/", looking
704        in remotes or heads as specified by the argument.  Return
705        a dictionary of { branch: revision } for each one found.
706        The branch names are the short names, without any
707        "p4/" prefix."""
708
709     branches = {}
710
711     cmdline = "git rev-parse --symbolic "
712     if branchesAreInRemotes:
713         cmdline += "--remotes"
714     else:
715         cmdline += "--branches"
716
717     for line in read_pipe_lines(cmdline):
718         line = line.strip()
719
720         # only import to p4/
721         if not line.startswith('p4/'):
722             continue
723         # special symbolic ref to p4/master
724         if line == "p4/HEAD":
725             continue
726
727         # strip off p4/ prefix
728         branch = line[len("p4/"):]
729
730         branches[branch] = parseRevision(line)
731
732     return branches
733
734 def branch_exists(branch):
735     """Make sure that the given ref name really exists."""
736
737     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
738     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
739     out, _ = p.communicate()
740     if p.returncode:
741         return False
742     # expect exactly one line of output: the branch name
743     return out.rstrip() == branch
744
745 def findUpstreamBranchPoint(head = "HEAD"):
746     branches = p4BranchesInGit()
747     # map from depot-path to branch name
748     branchByDepotPath = {}
749     for branch in branches.keys():
750         tip = branches[branch]
751         log = extractLogMessageFromGitCommit(tip)
752         settings = extractSettingsGitLog(log)
753         if settings.has_key("depot-paths"):
754             paths = ",".join(settings["depot-paths"])
755             branchByDepotPath[paths] = "remotes/p4/" + branch
756
757     settings = None
758     parent = 0
759     while parent < 65535:
760         commit = head + "~%s" % parent
761         log = extractLogMessageFromGitCommit(commit)
762         settings = extractSettingsGitLog(log)
763         if settings.has_key("depot-paths"):
764             paths = ",".join(settings["depot-paths"])
765             if branchByDepotPath.has_key(paths):
766                 return [branchByDepotPath[paths], settings]
767
768         parent = parent + 1
769
770     return ["", settings]
771
772 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
773     if not silent:
774         print ("Creating/updating branch(es) in %s based on origin branch(es)"
775                % localRefPrefix)
776
777     originPrefix = "origin/p4/"
778
779     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
780         line = line.strip()
781         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
782             continue
783
784         headName = line[len(originPrefix):]
785         remoteHead = localRefPrefix + headName
786         originHead = line
787
788         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
789         if (not original.has_key('depot-paths')
790             or not original.has_key('change')):
791             continue
792
793         update = False
794         if not gitBranchExists(remoteHead):
795             if verbose:
796                 print "creating %s" % remoteHead
797             update = True
798         else:
799             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
800             if settings.has_key('change') > 0:
801                 if settings['depot-paths'] == original['depot-paths']:
802                     originP4Change = int(original['change'])
803                     p4Change = int(settings['change'])
804                     if originP4Change > p4Change:
805                         print ("%s (%s) is newer than %s (%s). "
806                                "Updating p4 branch from origin."
807                                % (originHead, originP4Change,
808                                   remoteHead, p4Change))
809                         update = True
810                 else:
811                     print ("Ignoring: %s was imported from %s while "
812                            "%s was imported from %s"
813                            % (originHead, ','.join(original['depot-paths']),
814                               remoteHead, ','.join(settings['depot-paths'])))
815
816         if update:
817             system("git update-ref %s %s" % (remoteHead, originHead))
818
819 def originP4BranchesExist():
820         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
821
822
823 def p4ParseNumericChangeRange(parts):
824     changeStart = int(parts[0][1:])
825     if parts[1] == '#head':
826         changeEnd = p4_last_change()
827     else:
828         changeEnd = int(parts[1])
829
830     return (changeStart, changeEnd)
831
832 def chooseBlockSize(blockSize):
833     if blockSize:
834         return blockSize
835     else:
836         return defaultBlockSize
837
838 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
839     assert depotPaths
840
841     # Parse the change range into start and end. Try to find integer
842     # revision ranges as these can be broken up into blocks to avoid
843     # hitting server-side limits (maxrows, maxscanresults). But if
844     # that doesn't work, fall back to using the raw revision specifier
845     # strings, without using block mode.
846
847     if changeRange is None or changeRange == '':
848         changeStart = 1
849         changeEnd = p4_last_change()
850         block_size = chooseBlockSize(requestedBlockSize)
851     else:
852         parts = changeRange.split(',')
853         assert len(parts) == 2
854         try:
855             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
856             block_size = chooseBlockSize(requestedBlockSize)
857         except:
858             changeStart = parts[0][1:]
859             changeEnd = parts[1]
860             if requestedBlockSize:
861                 die("cannot use --changes-block-size with non-numeric revisions")
862             block_size = None
863
864     changes = set()
865
866     # Retrieve changes a block at a time, to prevent running
867     # into a MaxResults/MaxScanRows error from the server.
868
869     while True:
870         cmd = ['changes']
871
872         if block_size:
873             end = min(changeEnd, changeStart + block_size)
874             revisionRange = "%d,%d" % (changeStart, end)
875         else:
876             revisionRange = "%s,%s" % (changeStart, changeEnd)
877
878         for p in depotPaths:
879             cmd += ["%s...@%s" % (p, revisionRange)]
880
881         # Insert changes in chronological order
882         for line in reversed(p4_read_pipe_lines(cmd)):
883             changes.add(int(line.split(" ")[1]))
884
885         if not block_size:
886             break
887
888         if end >= changeEnd:
889             break
890
891         changeStart = end + 1
892
893     changes = sorted(changes)
894     return changes
895
896 def p4PathStartsWith(path, prefix):
897     # This method tries to remedy a potential mixed-case issue:
898     #
899     # If UserA adds  //depot/DirA/file1
900     # and UserB adds //depot/dira/file2
901     #
902     # we may or may not have a problem. If you have core.ignorecase=true,
903     # we treat DirA and dira as the same directory
904     if gitConfigBool("core.ignorecase"):
905         return path.lower().startswith(prefix.lower())
906     return path.startswith(prefix)
907
908 def getClientSpec():
909     """Look at the p4 client spec, create a View() object that contains
910        all the mappings, and return it."""
911
912     specList = p4CmdList("client -o")
913     if len(specList) != 1:
914         die('Output from "client -o" is %d lines, expecting 1' %
915             len(specList))
916
917     # dictionary of all client parameters
918     entry = specList[0]
919
920     # the //client/ name
921     client_name = entry["Client"]
922
923     # just the keys that start with "View"
924     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
925
926     # hold this new View
927     view = View(client_name)
928
929     # append the lines, in order, to the view
930     for view_num in range(len(view_keys)):
931         k = "View%d" % view_num
932         if k not in view_keys:
933             die("Expected view key %s missing" % k)
934         view.append(entry[k])
935
936     return view
937
938 def getClientRoot():
939     """Grab the client directory."""
940
941     output = p4CmdList("client -o")
942     if len(output) != 1:
943         die('Output from "client -o" is %d lines, expecting 1' % len(output))
944
945     entry = output[0]
946     if "Root" not in entry:
947         die('Client has no "Root"')
948
949     return entry["Root"]
950
951 #
952 # P4 wildcards are not allowed in filenames.  P4 complains
953 # if you simply add them, but you can force it with "-f", in
954 # which case it translates them into %xx encoding internally.
955 #
956 def wildcard_decode(path):
957     # Search for and fix just these four characters.  Do % last so
958     # that fixing it does not inadvertently create new %-escapes.
959     # Cannot have * in a filename in windows; untested as to
960     # what p4 would do in such a case.
961     if not platform.system() == "Windows":
962         path = path.replace("%2A", "*")
963     path = path.replace("%23", "#") \
964                .replace("%40", "@") \
965                .replace("%25", "%")
966     return path
967
968 def wildcard_encode(path):
969     # do % first to avoid double-encoding the %s introduced here
970     path = path.replace("%", "%25") \
971                .replace("*", "%2A") \
972                .replace("#", "%23") \
973                .replace("@", "%40")
974     return path
975
976 def wildcard_present(path):
977     m = re.search("[*#@%]", path)
978     return m is not None
979
980 class LargeFileSystem(object):
981     """Base class for large file system support."""
982
983     def __init__(self, writeToGitStream):
984         self.largeFiles = set()
985         self.writeToGitStream = writeToGitStream
986
987     def generatePointer(self, cloneDestination, contentFile):
988         """Return the content of a pointer file that is stored in Git instead of
989            the actual content."""
990         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
991
992     def pushFile(self, localLargeFile):
993         """Push the actual content which is not stored in the Git repository to
994            a server."""
995         assert False, "Method 'pushFile' required in " + self.__class__.__name__
996
997     def hasLargeFileExtension(self, relPath):
998         return reduce(
999             lambda a, b: a or b,
1000             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1001             False
1002         )
1003
1004     def generateTempFile(self, contents):
1005         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1006         for d in contents:
1007             contentFile.write(d)
1008         contentFile.close()
1009         return contentFile.name
1010
1011     def exceedsLargeFileThreshold(self, relPath, contents):
1012         if gitConfigInt('git-p4.largeFileThreshold'):
1013             contentsSize = sum(len(d) for d in contents)
1014             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1015                 return True
1016         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1017             contentsSize = sum(len(d) for d in contents)
1018             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1019                 return False
1020             contentTempFile = self.generateTempFile(contents)
1021             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1022             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1023             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1024             zf.close()
1025             compressedContentsSize = zf.infolist()[0].compress_size
1026             os.remove(contentTempFile)
1027             os.remove(compressedContentFile.name)
1028             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1029                 return True
1030         return False
1031
1032     def addLargeFile(self, relPath):
1033         self.largeFiles.add(relPath)
1034
1035     def removeLargeFile(self, relPath):
1036         self.largeFiles.remove(relPath)
1037
1038     def isLargeFile(self, relPath):
1039         return relPath in self.largeFiles
1040
1041     def processContent(self, git_mode, relPath, contents):
1042         """Processes the content of git fast import. This method decides if a
1043            file is stored in the large file system and handles all necessary
1044            steps."""
1045         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1046             contentTempFile = self.generateTempFile(contents)
1047             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1048             if pointer_git_mode:
1049                 git_mode = pointer_git_mode
1050             if localLargeFile:
1051                 # Move temp file to final location in large file system
1052                 largeFileDir = os.path.dirname(localLargeFile)
1053                 if not os.path.isdir(largeFileDir):
1054                     os.makedirs(largeFileDir)
1055                 shutil.move(contentTempFile, localLargeFile)
1056                 self.addLargeFile(relPath)
1057                 if gitConfigBool('git-p4.largeFilePush'):
1058                     self.pushFile(localLargeFile)
1059                 if verbose:
1060                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1061         return (git_mode, contents)
1062
1063 class MockLFS(LargeFileSystem):
1064     """Mock large file system for testing."""
1065
1066     def generatePointer(self, contentFile):
1067         """The pointer content is the original content prefixed with "pointer-".
1068            The local filename of the large file storage is derived from the file content.
1069            """
1070         with open(contentFile, 'r') as f:
1071             content = next(f)
1072             gitMode = '100644'
1073             pointerContents = 'pointer-' + content
1074             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1075             return (gitMode, pointerContents, localLargeFile)
1076
1077     def pushFile(self, localLargeFile):
1078         """The remote filename of the large file storage is the same as the local
1079            one but in a different directory.
1080            """
1081         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1082         if not os.path.exists(remotePath):
1083             os.makedirs(remotePath)
1084         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1085
1086 class GitLFS(LargeFileSystem):
1087     """Git LFS as backend for the git-p4 large file system.
1088        See https://git-lfs.github.com/ for details."""
1089
1090     def __init__(self, *args):
1091         LargeFileSystem.__init__(self, *args)
1092         self.baseGitAttributes = []
1093
1094     def generatePointer(self, contentFile):
1095         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1096            mode and content which is stored in the Git repository instead of
1097            the actual content. Return also the new location of the actual
1098            content.
1099            """
1100         if os.path.getsize(contentFile) == 0:
1101             return (None, '', None)
1102
1103         pointerProcess = subprocess.Popen(
1104             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1105             stdout=subprocess.PIPE
1106         )
1107         pointerFile = pointerProcess.stdout.read()
1108         if pointerProcess.wait():
1109             os.remove(contentFile)
1110             die('git-lfs pointer command failed. Did you install the extension?')
1111
1112         # Git LFS removed the preamble in the output of the 'pointer' command
1113         # starting from version 1.2.0. Check for the preamble here to support
1114         # earlier versions.
1115         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1116         if pointerFile.startswith('Git LFS pointer for'):
1117             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1118
1119         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1120         localLargeFile = os.path.join(
1121             os.getcwd(),
1122             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1123             oid,
1124         )
1125         # LFS Spec states that pointer files should not have the executable bit set.
1126         gitMode = '100644'
1127         return (gitMode, pointerFile, localLargeFile)
1128
1129     def pushFile(self, localLargeFile):
1130         uploadProcess = subprocess.Popen(
1131             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1132         )
1133         if uploadProcess.wait():
1134             die('git-lfs push command failed. Did you define a remote?')
1135
1136     def generateGitAttributes(self):
1137         return (
1138             self.baseGitAttributes +
1139             [
1140                 '\n',
1141                 '#\n',
1142                 '# Git LFS (see https://git-lfs.github.com/)\n',
1143                 '#\n',
1144             ] +
1145             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1146                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1147             ] +
1148             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1149                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1150             ]
1151         )
1152
1153     def addLargeFile(self, relPath):
1154         LargeFileSystem.addLargeFile(self, relPath)
1155         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1156
1157     def removeLargeFile(self, relPath):
1158         LargeFileSystem.removeLargeFile(self, relPath)
1159         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1160
1161     def processContent(self, git_mode, relPath, contents):
1162         if relPath == '.gitattributes':
1163             self.baseGitAttributes = contents
1164             return (git_mode, self.generateGitAttributes())
1165         else:
1166             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1167
1168 class Command:
1169     def __init__(self):
1170         self.usage = "usage: %prog [options]"
1171         self.needsGit = True
1172         self.verbose = False
1173
1174 class P4UserMap:
1175     def __init__(self):
1176         self.userMapFromPerforceServer = False
1177         self.myP4UserId = None
1178
1179     def p4UserId(self):
1180         if self.myP4UserId:
1181             return self.myP4UserId
1182
1183         results = p4CmdList("user -o")
1184         for r in results:
1185             if r.has_key('User'):
1186                 self.myP4UserId = r['User']
1187                 return r['User']
1188         die("Could not find your p4 user id")
1189
1190     def p4UserIsMe(self, p4User):
1191         # return True if the given p4 user is actually me
1192         me = self.p4UserId()
1193         if not p4User or p4User != me:
1194             return False
1195         else:
1196             return True
1197
1198     def getUserCacheFilename(self):
1199         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1200         return home + "/.gitp4-usercache.txt"
1201
1202     def getUserMapFromPerforceServer(self):
1203         if self.userMapFromPerforceServer:
1204             return
1205         self.users = {}
1206         self.emails = {}
1207
1208         for output in p4CmdList("users"):
1209             if not output.has_key("User"):
1210                 continue
1211             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1212             self.emails[output["Email"]] = output["User"]
1213
1214         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1215         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1216             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1217             if mapUser and len(mapUser[0]) == 3:
1218                 user = mapUser[0][0]
1219                 fullname = mapUser[0][1]
1220                 email = mapUser[0][2]
1221                 self.users[user] = fullname + " <" + email + ">"
1222                 self.emails[email] = user
1223
1224         s = ''
1225         for (key, val) in self.users.items():
1226             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1227
1228         open(self.getUserCacheFilename(), "wb").write(s)
1229         self.userMapFromPerforceServer = True
1230
1231     def loadUserMapFromCache(self):
1232         self.users = {}
1233         self.userMapFromPerforceServer = False
1234         try:
1235             cache = open(self.getUserCacheFilename(), "rb")
1236             lines = cache.readlines()
1237             cache.close()
1238             for line in lines:
1239                 entry = line.strip().split("\t")
1240                 self.users[entry[0]] = entry[1]
1241         except IOError:
1242             self.getUserMapFromPerforceServer()
1243
1244 class P4Debug(Command):
1245     def __init__(self):
1246         Command.__init__(self)
1247         self.options = []
1248         self.description = "A tool to debug the output of p4 -G."
1249         self.needsGit = False
1250
1251     def run(self, args):
1252         j = 0
1253         for output in p4CmdList(args):
1254             print 'Element: %d' % j
1255             j += 1
1256             print output
1257         return True
1258
1259 class P4RollBack(Command):
1260     def __init__(self):
1261         Command.__init__(self)
1262         self.options = [
1263             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1264         ]
1265         self.description = "A tool to debug the multi-branch import. Don't use :)"
1266         self.rollbackLocalBranches = False
1267
1268     def run(self, args):
1269         if len(args) != 1:
1270             return False
1271         maxChange = int(args[0])
1272
1273         if "p4ExitCode" in p4Cmd("changes -m 1"):
1274             die("Problems executing p4");
1275
1276         if self.rollbackLocalBranches:
1277             refPrefix = "refs/heads/"
1278             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1279         else:
1280             refPrefix = "refs/remotes/"
1281             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1282
1283         for line in lines:
1284             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1285                 line = line.strip()
1286                 ref = refPrefix + line
1287                 log = extractLogMessageFromGitCommit(ref)
1288                 settings = extractSettingsGitLog(log)
1289
1290                 depotPaths = settings['depot-paths']
1291                 change = settings['change']
1292
1293                 changed = False
1294
1295                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1296                                                            for p in depotPaths]))) == 0:
1297                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1298                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1299                     continue
1300
1301                 while change and int(change) > maxChange:
1302                     changed = True
1303                     if self.verbose:
1304                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1305                     system("git update-ref %s \"%s^\"" % (ref, ref))
1306                     log = extractLogMessageFromGitCommit(ref)
1307                     settings =  extractSettingsGitLog(log)
1308
1309
1310                     depotPaths = settings['depot-paths']
1311                     change = settings['change']
1312
1313                 if changed:
1314                     print "%s rewound to %s" % (ref, change)
1315
1316         return True
1317
1318 class P4Submit(Command, P4UserMap):
1319
1320     conflict_behavior_choices = ("ask", "skip", "quit")
1321
1322     def __init__(self):
1323         Command.__init__(self)
1324         P4UserMap.__init__(self)
1325         self.options = [
1326                 optparse.make_option("--origin", dest="origin"),
1327                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1328                 # preserve the user, requires relevant p4 permissions
1329                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1330                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1331                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1332                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1333                 optparse.make_option("--conflict", dest="conflict_behavior",
1334                                      choices=self.conflict_behavior_choices),
1335                 optparse.make_option("--branch", dest="branch"),
1336                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1337                                      help="Shelve instead of submit. Shelved files are reverted, "
1338                                      "restoring the workspace to the state before the shelve"),
1339                 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1340                                      metavar="CHANGELIST",
1341                                      help="update an existing shelved changelist, implies --shelve")
1342         ]
1343         self.description = "Submit changes from git to the perforce depot."
1344         self.usage += " [name of git branch to submit into perforce depot]"
1345         self.origin = ""
1346         self.detectRenames = False
1347         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1348         self.dry_run = False
1349         self.shelve = False
1350         self.update_shelve = None
1351         self.prepare_p4_only = False
1352         self.conflict_behavior = None
1353         self.isWindows = (platform.system() == "Windows")
1354         self.exportLabels = False
1355         self.p4HasMoveCommand = p4_has_move_command()
1356         self.branch = None
1357
1358         if gitConfig('git-p4.largeFileSystem'):
1359             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1360
1361     def check(self):
1362         if len(p4CmdList("opened ...")) > 0:
1363             die("You have files opened with perforce! Close them before starting the sync.")
1364
1365     def separate_jobs_from_description(self, message):
1366         """Extract and return a possible Jobs field in the commit
1367            message.  It goes into a separate section in the p4 change
1368            specification.
1369
1370            A jobs line starts with "Jobs:" and looks like a new field
1371            in a form.  Values are white-space separated on the same
1372            line or on following lines that start with a tab.
1373
1374            This does not parse and extract the full git commit message
1375            like a p4 form.  It just sees the Jobs: line as a marker
1376            to pass everything from then on directly into the p4 form,
1377            but outside the description section.
1378
1379            Return a tuple (stripped log message, jobs string)."""
1380
1381         m = re.search(r'^Jobs:', message, re.MULTILINE)
1382         if m is None:
1383             return (message, None)
1384
1385         jobtext = message[m.start():]
1386         stripped_message = message[:m.start()].rstrip()
1387         return (stripped_message, jobtext)
1388
1389     def prepareLogMessage(self, template, message, jobs):
1390         """Edits the template returned from "p4 change -o" to insert
1391            the message in the Description field, and the jobs text in
1392            the Jobs field."""
1393         result = ""
1394
1395         inDescriptionSection = False
1396
1397         for line in template.split("\n"):
1398             if line.startswith("#"):
1399                 result += line + "\n"
1400                 continue
1401
1402             if inDescriptionSection:
1403                 if line.startswith("Files:") or line.startswith("Jobs:"):
1404                     inDescriptionSection = False
1405                     # insert Jobs section
1406                     if jobs:
1407                         result += jobs + "\n"
1408                 else:
1409                     continue
1410             else:
1411                 if line.startswith("Description:"):
1412                     inDescriptionSection = True
1413                     line += "\n"
1414                     for messageLine in message.split("\n"):
1415                         line += "\t" + messageLine + "\n"
1416
1417             result += line + "\n"
1418
1419         return result
1420
1421     def patchRCSKeywords(self, file, pattern):
1422         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1423         (handle, outFileName) = tempfile.mkstemp(dir='.')
1424         try:
1425             outFile = os.fdopen(handle, "w+")
1426             inFile = open(file, "r")
1427             regexp = re.compile(pattern, re.VERBOSE)
1428             for line in inFile.readlines():
1429                 line = regexp.sub(r'$\1$', line)
1430                 outFile.write(line)
1431             inFile.close()
1432             outFile.close()
1433             # Forcibly overwrite the original file
1434             os.unlink(file)
1435             shutil.move(outFileName, file)
1436         except:
1437             # cleanup our temporary file
1438             os.unlink(outFileName)
1439             print "Failed to strip RCS keywords in %s" % file
1440             raise
1441
1442         print "Patched up RCS keywords in %s" % file
1443
1444     def p4UserForCommit(self,id):
1445         # Return the tuple (perforce user,git email) for a given git commit id
1446         self.getUserMapFromPerforceServer()
1447         gitEmail = read_pipe(["git", "log", "--max-count=1",
1448                               "--format=%ae", id])
1449         gitEmail = gitEmail.strip()
1450         if not self.emails.has_key(gitEmail):
1451             return (None,gitEmail)
1452         else:
1453             return (self.emails[gitEmail],gitEmail)
1454
1455     def checkValidP4Users(self,commits):
1456         # check if any git authors cannot be mapped to p4 users
1457         for id in commits:
1458             (user,email) = self.p4UserForCommit(id)
1459             if not user:
1460                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1461                 if gitConfigBool("git-p4.allowMissingP4Users"):
1462                     print "%s" % msg
1463                 else:
1464                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1465
1466     def lastP4Changelist(self):
1467         # Get back the last changelist number submitted in this client spec. This
1468         # then gets used to patch up the username in the change. If the same
1469         # client spec is being used by multiple processes then this might go
1470         # wrong.
1471         results = p4CmdList("client -o")        # find the current client
1472         client = None
1473         for r in results:
1474             if r.has_key('Client'):
1475                 client = r['Client']
1476                 break
1477         if not client:
1478             die("could not get client spec")
1479         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1480         for r in results:
1481             if r.has_key('change'):
1482                 return r['change']
1483         die("Could not get changelist number for last submit - cannot patch up user details")
1484
1485     def modifyChangelistUser(self, changelist, newUser):
1486         # fixup the user field of a changelist after it has been submitted.
1487         changes = p4CmdList("change -o %s" % changelist)
1488         if len(changes) != 1:
1489             die("Bad output from p4 change modifying %s to user %s" %
1490                 (changelist, newUser))
1491
1492         c = changes[0]
1493         if c['User'] == newUser: return   # nothing to do
1494         c['User'] = newUser
1495         input = marshal.dumps(c)
1496
1497         result = p4CmdList("change -f -i", stdin=input)
1498         for r in result:
1499             if r.has_key('code'):
1500                 if r['code'] == 'error':
1501                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1502             if r.has_key('data'):
1503                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1504                 return
1505         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1506
1507     def canChangeChangelists(self):
1508         # check to see if we have p4 admin or super-user permissions, either of
1509         # which are required to modify changelists.
1510         results = p4CmdList(["protects", self.depotPath])
1511         for r in results:
1512             if r.has_key('perm'):
1513                 if r['perm'] == 'admin':
1514                     return 1
1515                 if r['perm'] == 'super':
1516                     return 1
1517         return 0
1518
1519     def prepareSubmitTemplate(self, changelist=None):
1520         """Run "p4 change -o" to grab a change specification template.
1521            This does not use "p4 -G", as it is nice to keep the submission
1522            template in original order, since a human might edit it.
1523
1524            Remove lines in the Files section that show changes to files
1525            outside the depot path we're committing into."""
1526
1527         [upstream, settings] = findUpstreamBranchPoint()
1528
1529         template = ""
1530         inFilesSection = False
1531         args = ['change', '-o']
1532         if changelist:
1533             args.append(str(changelist))
1534
1535         for line in p4_read_pipe_lines(args):
1536             if line.endswith("\r\n"):
1537                 line = line[:-2] + "\n"
1538             if inFilesSection:
1539                 if line.startswith("\t"):
1540                     # path starts and ends with a tab
1541                     path = line[1:]
1542                     lastTab = path.rfind("\t")
1543                     if lastTab != -1:
1544                         path = path[:lastTab]
1545                         if settings.has_key('depot-paths'):
1546                             if not [p for p in settings['depot-paths']
1547                                     if p4PathStartsWith(path, p)]:
1548                                 continue
1549                         else:
1550                             if not p4PathStartsWith(path, self.depotPath):
1551                                 continue
1552                 else:
1553                     inFilesSection = False
1554             else:
1555                 if line.startswith("Files:"):
1556                     inFilesSection = True
1557
1558             template += line
1559
1560         return template
1561
1562     def edit_template(self, template_file):
1563         """Invoke the editor to let the user change the submission
1564            message.  Return true if okay to continue with the submit."""
1565
1566         # if configured to skip the editing part, just submit
1567         if gitConfigBool("git-p4.skipSubmitEdit"):
1568             return True
1569
1570         # look at the modification time, to check later if the user saved
1571         # the file
1572         mtime = os.stat(template_file).st_mtime
1573
1574         # invoke the editor
1575         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1576             editor = os.environ.get("P4EDITOR")
1577         else:
1578             editor = read_pipe("git var GIT_EDITOR").strip()
1579         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1580
1581         # If the file was not saved, prompt to see if this patch should
1582         # be skipped.  But skip this verification step if configured so.
1583         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1584             return True
1585
1586         # modification time updated means user saved the file
1587         if os.stat(template_file).st_mtime > mtime:
1588             return True
1589
1590         while True:
1591             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1592             if response == 'y':
1593                 return True
1594             if response == 'n':
1595                 return False
1596
1597     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1598         # diff
1599         if os.environ.has_key("P4DIFF"):
1600             del(os.environ["P4DIFF"])
1601         diff = ""
1602         for editedFile in editedFiles:
1603             diff += p4_read_pipe(['diff', '-du',
1604                                   wildcard_encode(editedFile)])
1605
1606         # new file diff
1607         newdiff = ""
1608         for newFile in filesToAdd:
1609             newdiff += "==== new file ====\n"
1610             newdiff += "--- /dev/null\n"
1611             newdiff += "+++ %s\n" % newFile
1612
1613             is_link = os.path.islink(newFile)
1614             expect_link = newFile in symlinks
1615
1616             if is_link and expect_link:
1617                 newdiff += "+%s\n" % os.readlink(newFile)
1618             else:
1619                 f = open(newFile, "r")
1620                 for line in f.readlines():
1621                     newdiff += "+" + line
1622                 f.close()
1623
1624         return (diff + newdiff).replace('\r\n', '\n')
1625
1626     def applyCommit(self, id):
1627         """Apply one commit, return True if it succeeded."""
1628
1629         print "Applying", read_pipe(["git", "show", "-s",
1630                                      "--format=format:%h %s", id])
1631
1632         (p4User, gitEmail) = self.p4UserForCommit(id)
1633
1634         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1635         filesToAdd = set()
1636         filesToChangeType = set()
1637         filesToDelete = set()
1638         editedFiles = set()
1639         pureRenameCopy = set()
1640         symlinks = set()
1641         filesToChangeExecBit = {}
1642         all_files = list()
1643
1644         for line in diff:
1645             diff = parseDiffTreeEntry(line)
1646             modifier = diff['status']
1647             path = diff['src']
1648             all_files.append(path)
1649
1650             if modifier == "M":
1651                 p4_edit(path)
1652                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1653                     filesToChangeExecBit[path] = diff['dst_mode']
1654                 editedFiles.add(path)
1655             elif modifier == "A":
1656                 filesToAdd.add(path)
1657                 filesToChangeExecBit[path] = diff['dst_mode']
1658                 if path in filesToDelete:
1659                     filesToDelete.remove(path)
1660
1661                 dst_mode = int(diff['dst_mode'], 8)
1662                 if dst_mode == 0120000:
1663                     symlinks.add(path)
1664
1665             elif modifier == "D":
1666                 filesToDelete.add(path)
1667                 if path in filesToAdd:
1668                     filesToAdd.remove(path)
1669             elif modifier == "C":
1670                 src, dest = diff['src'], diff['dst']
1671                 p4_integrate(src, dest)
1672                 pureRenameCopy.add(dest)
1673                 if diff['src_sha1'] != diff['dst_sha1']:
1674                     p4_edit(dest)
1675                     pureRenameCopy.discard(dest)
1676                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1677                     p4_edit(dest)
1678                     pureRenameCopy.discard(dest)
1679                     filesToChangeExecBit[dest] = diff['dst_mode']
1680                 if self.isWindows:
1681                     # turn off read-only attribute
1682                     os.chmod(dest, stat.S_IWRITE)
1683                 os.unlink(dest)
1684                 editedFiles.add(dest)
1685             elif modifier == "R":
1686                 src, dest = diff['src'], diff['dst']
1687                 if self.p4HasMoveCommand:
1688                     p4_edit(src)        # src must be open before move
1689                     p4_move(src, dest)  # opens for (move/delete, move/add)
1690                 else:
1691                     p4_integrate(src, dest)
1692                     if diff['src_sha1'] != diff['dst_sha1']:
1693                         p4_edit(dest)
1694                     else:
1695                         pureRenameCopy.add(dest)
1696                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1697                     if not self.p4HasMoveCommand:
1698                         p4_edit(dest)   # with move: already open, writable
1699                     filesToChangeExecBit[dest] = diff['dst_mode']
1700                 if not self.p4HasMoveCommand:
1701                     if self.isWindows:
1702                         os.chmod(dest, stat.S_IWRITE)
1703                     os.unlink(dest)
1704                     filesToDelete.add(src)
1705                 editedFiles.add(dest)
1706             elif modifier == "T":
1707                 filesToChangeType.add(path)
1708             else:
1709                 die("unknown modifier %s for %s" % (modifier, path))
1710
1711         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1712         patchcmd = diffcmd + " | git apply "
1713         tryPatchCmd = patchcmd + "--check -"
1714         applyPatchCmd = patchcmd + "--check --apply -"
1715         patch_succeeded = True
1716
1717         if os.system(tryPatchCmd) != 0:
1718             fixed_rcs_keywords = False
1719             patch_succeeded = False
1720             print "Unfortunately applying the change failed!"
1721
1722             # Patch failed, maybe it's just RCS keyword woes. Look through
1723             # the patch to see if that's possible.
1724             if gitConfigBool("git-p4.attemptRCSCleanup"):
1725                 file = None
1726                 pattern = None
1727                 kwfiles = {}
1728                 for file in editedFiles | filesToDelete:
1729                     # did this file's delta contain RCS keywords?
1730                     pattern = p4_keywords_regexp_for_file(file)
1731
1732                     if pattern:
1733                         # this file is a possibility...look for RCS keywords.
1734                         regexp = re.compile(pattern, re.VERBOSE)
1735                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1736                             if regexp.search(line):
1737                                 if verbose:
1738                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1739                                 kwfiles[file] = pattern
1740                                 break
1741
1742                 for file in kwfiles:
1743                     if verbose:
1744                         print "zapping %s with %s" % (line,pattern)
1745                     # File is being deleted, so not open in p4.  Must
1746                     # disable the read-only bit on windows.
1747                     if self.isWindows and file not in editedFiles:
1748                         os.chmod(file, stat.S_IWRITE)
1749                     self.patchRCSKeywords(file, kwfiles[file])
1750                     fixed_rcs_keywords = True
1751
1752             if fixed_rcs_keywords:
1753                 print "Retrying the patch with RCS keywords cleaned up"
1754                 if os.system(tryPatchCmd) == 0:
1755                     patch_succeeded = True
1756
1757         if not patch_succeeded:
1758             for f in editedFiles:
1759                 p4_revert(f)
1760             return False
1761
1762         #
1763         # Apply the patch for real, and do add/delete/+x handling.
1764         #
1765         system(applyPatchCmd)
1766
1767         for f in filesToChangeType:
1768             p4_edit(f, "-t", "auto")
1769         for f in filesToAdd:
1770             p4_add(f)
1771         for f in filesToDelete:
1772             p4_revert(f)
1773             p4_delete(f)
1774
1775         # Set/clear executable bits
1776         for f in filesToChangeExecBit.keys():
1777             mode = filesToChangeExecBit[f]
1778             setP4ExecBit(f, mode)
1779
1780         if self.update_shelve:
1781             print("all_files = %s" % str(all_files))
1782             p4_reopen_in_change(self.update_shelve, all_files)
1783
1784         #
1785         # Build p4 change description, starting with the contents
1786         # of the git commit message.
1787         #
1788         logMessage = extractLogMessageFromGitCommit(id)
1789         logMessage = logMessage.strip()
1790         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1791
1792         template = self.prepareSubmitTemplate(self.update_shelve)
1793         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1794
1795         if self.preserveUser:
1796            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1797
1798         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1799             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1800             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1801             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1802
1803         separatorLine = "######## everything below this line is just the diff #######\n"
1804         if not self.prepare_p4_only:
1805             submitTemplate += separatorLine
1806             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1807
1808         (handle, fileName) = tempfile.mkstemp()
1809         tmpFile = os.fdopen(handle, "w+b")
1810         if self.isWindows:
1811             submitTemplate = submitTemplate.replace("\n", "\r\n")
1812         tmpFile.write(submitTemplate)
1813         tmpFile.close()
1814
1815         if self.prepare_p4_only:
1816             #
1817             # Leave the p4 tree prepared, and the submit template around
1818             # and let the user decide what to do next
1819             #
1820             print
1821             print "P4 workspace prepared for submission."
1822             print "To submit or revert, go to client workspace"
1823             print "  " + self.clientPath
1824             print
1825             print "To submit, use \"p4 submit\" to write a new description,"
1826             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1827                   " \"git p4\"." % fileName
1828             print "You can delete the file \"%s\" when finished." % fileName
1829
1830             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1831                 print "To preserve change ownership by user %s, you must\n" \
1832                       "do \"p4 change -f <change>\" after submitting and\n" \
1833                       "edit the User field."
1834             if pureRenameCopy:
1835                 print "After submitting, renamed files must be re-synced."
1836                 print "Invoke \"p4 sync -f\" on each of these files:"
1837                 for f in pureRenameCopy:
1838                     print "  " + f
1839
1840             print
1841             print "To revert the changes, use \"p4 revert ...\", and delete"
1842             print "the submit template file \"%s\"" % fileName
1843             if filesToAdd:
1844                 print "Since the commit adds new files, they must be deleted:"
1845                 for f in filesToAdd:
1846                     print "  " + f
1847             print
1848             return True
1849
1850         #
1851         # Let the user edit the change description, then submit it.
1852         #
1853         submitted = False
1854
1855         try:
1856             if self.edit_template(fileName):
1857                 # read the edited message and submit
1858                 tmpFile = open(fileName, "rb")
1859                 message = tmpFile.read()
1860                 tmpFile.close()
1861                 if self.isWindows:
1862                     message = message.replace("\r\n", "\n")
1863                 submitTemplate = message[:message.index(separatorLine)]
1864
1865                 if self.update_shelve:
1866                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1867                 elif self.shelve:
1868                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1869                 else:
1870                     p4_write_pipe(['submit', '-i'], submitTemplate)
1871                     # The rename/copy happened by applying a patch that created a
1872                     # new file.  This leaves it writable, which confuses p4.
1873                     for f in pureRenameCopy:
1874                         p4_sync(f, "-f")
1875
1876                 if self.preserveUser:
1877                     if p4User:
1878                         # Get last changelist number. Cannot easily get it from
1879                         # the submit command output as the output is
1880                         # unmarshalled.
1881                         changelist = self.lastP4Changelist()
1882                         self.modifyChangelistUser(changelist, p4User)
1883
1884                 submitted = True
1885
1886         finally:
1887             # skip this patch
1888             if not submitted or self.shelve:
1889                 if self.shelve:
1890                     print ("Reverting shelved files.")
1891                 else:
1892                     print ("Submission cancelled, undoing p4 changes.")
1893                 for f in editedFiles | filesToDelete:
1894                     p4_revert(f)
1895                 for f in filesToAdd:
1896                     p4_revert(f)
1897                     os.remove(f)
1898
1899         os.remove(fileName)
1900         return submitted
1901
1902     # Export git tags as p4 labels. Create a p4 label and then tag
1903     # with that.
1904     def exportGitTags(self, gitTags):
1905         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1906         if len(validLabelRegexp) == 0:
1907             validLabelRegexp = defaultLabelRegexp
1908         m = re.compile(validLabelRegexp)
1909
1910         for name in gitTags:
1911
1912             if not m.match(name):
1913                 if verbose:
1914                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1915                 continue
1916
1917             # Get the p4 commit this corresponds to
1918             logMessage = extractLogMessageFromGitCommit(name)
1919             values = extractSettingsGitLog(logMessage)
1920
1921             if not values.has_key('change'):
1922                 # a tag pointing to something not sent to p4; ignore
1923                 if verbose:
1924                     print "git tag %s does not give a p4 commit" % name
1925                 continue
1926             else:
1927                 changelist = values['change']
1928
1929             # Get the tag details.
1930             inHeader = True
1931             isAnnotated = False
1932             body = []
1933             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1934                 l = l.strip()
1935                 if inHeader:
1936                     if re.match(r'tag\s+', l):
1937                         isAnnotated = True
1938                     elif re.match(r'\s*$', l):
1939                         inHeader = False
1940                         continue
1941                 else:
1942                     body.append(l)
1943
1944             if not isAnnotated:
1945                 body = ["lightweight tag imported by git p4\n"]
1946
1947             # Create the label - use the same view as the client spec we are using
1948             clientSpec = getClientSpec()
1949
1950             labelTemplate  = "Label: %s\n" % name
1951             labelTemplate += "Description:\n"
1952             for b in body:
1953                 labelTemplate += "\t" + b + "\n"
1954             labelTemplate += "View:\n"
1955             for depot_side in clientSpec.mappings:
1956                 labelTemplate += "\t%s\n" % depot_side
1957
1958             if self.dry_run:
1959                 print "Would create p4 label %s for tag" % name
1960             elif self.prepare_p4_only:
1961                 print "Not creating p4 label %s for tag due to option" \
1962                       " --prepare-p4-only" % name
1963             else:
1964                 p4_write_pipe(["label", "-i"], labelTemplate)
1965
1966                 # Use the label
1967                 p4_system(["tag", "-l", name] +
1968                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1969
1970                 if verbose:
1971                     print "created p4 label for tag %s" % name
1972
1973     def run(self, args):
1974         if len(args) == 0:
1975             self.master = currentGitBranch()
1976         elif len(args) == 1:
1977             self.master = args[0]
1978             if not branchExists(self.master):
1979                 die("Branch %s does not exist" % self.master)
1980         else:
1981             return False
1982
1983         if self.master:
1984             allowSubmit = gitConfig("git-p4.allowSubmit")
1985             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1986                 die("%s is not in git-p4.allowSubmit" % self.master)
1987
1988         [upstream, settings] = findUpstreamBranchPoint()
1989         self.depotPath = settings['depot-paths'][0]
1990         if len(self.origin) == 0:
1991             self.origin = upstream
1992
1993         if self.update_shelve:
1994             self.shelve = True
1995
1996         if self.preserveUser:
1997             if not self.canChangeChangelists():
1998                 die("Cannot preserve user names without p4 super-user or admin permissions")
1999
2000         # if not set from the command line, try the config file
2001         if self.conflict_behavior is None:
2002             val = gitConfig("git-p4.conflict")
2003             if val:
2004                 if val not in self.conflict_behavior_choices:
2005                     die("Invalid value '%s' for config git-p4.conflict" % val)
2006             else:
2007                 val = "ask"
2008             self.conflict_behavior = val
2009
2010         if self.verbose:
2011             print "Origin branch is " + self.origin
2012
2013         if len(self.depotPath) == 0:
2014             print "Internal error: cannot locate perforce depot path from existing branches"
2015             sys.exit(128)
2016
2017         self.useClientSpec = False
2018         if gitConfigBool("git-p4.useclientspec"):
2019             self.useClientSpec = True
2020         if self.useClientSpec:
2021             self.clientSpecDirs = getClientSpec()
2022
2023         # Check for the existence of P4 branches
2024         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2025
2026         if self.useClientSpec and not branchesDetected:
2027             # all files are relative to the client spec
2028             self.clientPath = getClientRoot()
2029         else:
2030             self.clientPath = p4Where(self.depotPath)
2031
2032         if self.clientPath == "":
2033             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2034
2035         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2036         self.oldWorkingDirectory = os.getcwd()
2037
2038         # ensure the clientPath exists
2039         new_client_dir = False
2040         if not os.path.exists(self.clientPath):
2041             new_client_dir = True
2042             os.makedirs(self.clientPath)
2043
2044         chdir(self.clientPath, is_client_path=True)
2045         if self.dry_run:
2046             print "Would synchronize p4 checkout in %s" % self.clientPath
2047         else:
2048             print "Synchronizing p4 checkout..."
2049             if new_client_dir:
2050                 # old one was destroyed, and maybe nobody told p4
2051                 p4_sync("...", "-f")
2052             else:
2053                 p4_sync("...")
2054         self.check()
2055
2056         commits = []
2057         if self.master:
2058             commitish = self.master
2059         else:
2060             commitish = 'HEAD'
2061
2062         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2063             commits.append(line.strip())
2064         commits.reverse()
2065
2066         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2067             self.checkAuthorship = False
2068         else:
2069             self.checkAuthorship = True
2070
2071         if self.preserveUser:
2072             self.checkValidP4Users(commits)
2073
2074         #
2075         # Build up a set of options to be passed to diff when
2076         # submitting each commit to p4.
2077         #
2078         if self.detectRenames:
2079             # command-line -M arg
2080             self.diffOpts = "-M"
2081         else:
2082             # If not explicitly set check the config variable
2083             detectRenames = gitConfig("git-p4.detectRenames")
2084
2085             if detectRenames.lower() == "false" or detectRenames == "":
2086                 self.diffOpts = ""
2087             elif detectRenames.lower() == "true":
2088                 self.diffOpts = "-M"
2089             else:
2090                 self.diffOpts = "-M%s" % detectRenames
2091
2092         # no command-line arg for -C or --find-copies-harder, just
2093         # config variables
2094         detectCopies = gitConfig("git-p4.detectCopies")
2095         if detectCopies.lower() == "false" or detectCopies == "":
2096             pass
2097         elif detectCopies.lower() == "true":
2098             self.diffOpts += " -C"
2099         else:
2100             self.diffOpts += " -C%s" % detectCopies
2101
2102         if gitConfigBool("git-p4.detectCopiesHarder"):
2103             self.diffOpts += " --find-copies-harder"
2104
2105         #
2106         # Apply the commits, one at a time.  On failure, ask if should
2107         # continue to try the rest of the patches, or quit.
2108         #
2109         if self.dry_run:
2110             print "Would apply"
2111         applied = []
2112         last = len(commits) - 1
2113         for i, commit in enumerate(commits):
2114             if self.dry_run:
2115                 print " ", read_pipe(["git", "show", "-s",
2116                                       "--format=format:%h %s", commit])
2117                 ok = True
2118             else:
2119                 ok = self.applyCommit(commit)
2120             if ok:
2121                 applied.append(commit)
2122             else:
2123                 if self.prepare_p4_only and i < last:
2124                     print "Processing only the first commit due to option" \
2125                           " --prepare-p4-only"
2126                     break
2127                 if i < last:
2128                     quit = False
2129                     while True:
2130                         # prompt for what to do, or use the option/variable
2131                         if self.conflict_behavior == "ask":
2132                             print "What do you want to do?"
2133                             response = raw_input("[s]kip this commit but apply"
2134                                                  " the rest, or [q]uit? ")
2135                             if not response:
2136                                 continue
2137                         elif self.conflict_behavior == "skip":
2138                             response = "s"
2139                         elif self.conflict_behavior == "quit":
2140                             response = "q"
2141                         else:
2142                             die("Unknown conflict_behavior '%s'" %
2143                                 self.conflict_behavior)
2144
2145                         if response[0] == "s":
2146                             print "Skipping this commit, but applying the rest"
2147                             break
2148                         if response[0] == "q":
2149                             print "Quitting"
2150                             quit = True
2151                             break
2152                     if quit:
2153                         break
2154
2155         chdir(self.oldWorkingDirectory)
2156         shelved_applied = "shelved" if self.shelve else "applied"
2157         if self.dry_run:
2158             pass
2159         elif self.prepare_p4_only:
2160             pass
2161         elif len(commits) == len(applied):
2162             print ("All commits {0}!".format(shelved_applied))
2163
2164             sync = P4Sync()
2165             if self.branch:
2166                 sync.branch = self.branch
2167             sync.run([])
2168
2169             rebase = P4Rebase()
2170             rebase.rebase()
2171
2172         else:
2173             if len(applied) == 0:
2174                 print ("No commits {0}.".format(shelved_applied))
2175             else:
2176                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2177                 for c in commits:
2178                     if c in applied:
2179                         star = "*"
2180                     else:
2181                         star = " "
2182                     print star, read_pipe(["git", "show", "-s",
2183                                            "--format=format:%h %s",  c])
2184                 print "You will have to do 'git p4 sync' and rebase."
2185
2186         if gitConfigBool("git-p4.exportLabels"):
2187             self.exportLabels = True
2188
2189         if self.exportLabels:
2190             p4Labels = getP4Labels(self.depotPath)
2191             gitTags = getGitTags()
2192
2193             missingGitTags = gitTags - p4Labels
2194             self.exportGitTags(missingGitTags)
2195
2196         # exit with error unless everything applied perfectly
2197         if len(commits) != len(applied):
2198                 sys.exit(1)
2199
2200         return True
2201
2202 class View(object):
2203     """Represent a p4 view ("p4 help views"), and map files in a
2204        repo according to the view."""
2205
2206     def __init__(self, client_name):
2207         self.mappings = []
2208         self.client_prefix = "//%s/" % client_name
2209         # cache results of "p4 where" to lookup client file locations
2210         self.client_spec_path_cache = {}
2211
2212     def append(self, view_line):
2213         """Parse a view line, splitting it into depot and client
2214            sides.  Append to self.mappings, preserving order.  This
2215            is only needed for tag creation."""
2216
2217         # Split the view line into exactly two words.  P4 enforces
2218         # structure on these lines that simplifies this quite a bit.
2219         #
2220         # Either or both words may be double-quoted.
2221         # Single quotes do not matter.
2222         # Double-quote marks cannot occur inside the words.
2223         # A + or - prefix is also inside the quotes.
2224         # There are no quotes unless they contain a space.
2225         # The line is already white-space stripped.
2226         # The two words are separated by a single space.
2227         #
2228         if view_line[0] == '"':
2229             # First word is double quoted.  Find its end.
2230             close_quote_index = view_line.find('"', 1)
2231             if close_quote_index <= 0:
2232                 die("No first-word closing quote found: %s" % view_line)
2233             depot_side = view_line[1:close_quote_index]
2234             # skip closing quote and space
2235             rhs_index = close_quote_index + 1 + 1
2236         else:
2237             space_index = view_line.find(" ")
2238             if space_index <= 0:
2239                 die("No word-splitting space found: %s" % view_line)
2240             depot_side = view_line[0:space_index]
2241             rhs_index = space_index + 1
2242
2243         # prefix + means overlay on previous mapping
2244         if depot_side.startswith("+"):
2245             depot_side = depot_side[1:]
2246
2247         # prefix - means exclude this path, leave out of mappings
2248         exclude = False
2249         if depot_side.startswith("-"):
2250             exclude = True
2251             depot_side = depot_side[1:]
2252
2253         if not exclude:
2254             self.mappings.append(depot_side)
2255
2256     def convert_client_path(self, clientFile):
2257         # chop off //client/ part to make it relative
2258         if not clientFile.startswith(self.client_prefix):
2259             die("No prefix '%s' on clientFile '%s'" %
2260                 (self.client_prefix, clientFile))
2261         return clientFile[len(self.client_prefix):]
2262
2263     def update_client_spec_path_cache(self, files):
2264         """ Caching file paths by "p4 where" batch query """
2265
2266         # List depot file paths exclude that already cached
2267         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2268
2269         if len(fileArgs) == 0:
2270             return  # All files in cache
2271
2272         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2273         for res in where_result:
2274             if "code" in res and res["code"] == "error":
2275                 # assume error is "... file(s) not in client view"
2276                 continue
2277             if "clientFile" not in res:
2278                 die("No clientFile in 'p4 where' output")
2279             if "unmap" in res:
2280                 # it will list all of them, but only one not unmap-ped
2281                 continue
2282             if gitConfigBool("core.ignorecase"):
2283                 res['depotFile'] = res['depotFile'].lower()
2284             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2285
2286         # not found files or unmap files set to ""
2287         for depotFile in fileArgs:
2288             if gitConfigBool("core.ignorecase"):
2289                 depotFile = depotFile.lower()
2290             if depotFile not in self.client_spec_path_cache:
2291                 self.client_spec_path_cache[depotFile] = ""
2292
2293     def map_in_client(self, depot_path):
2294         """Return the relative location in the client where this
2295            depot file should live.  Returns "" if the file should
2296            not be mapped in the client."""
2297
2298         if gitConfigBool("core.ignorecase"):
2299             depot_path = depot_path.lower()
2300
2301         if depot_path in self.client_spec_path_cache:
2302             return self.client_spec_path_cache[depot_path]
2303
2304         die( "Error: %s is not found in client spec path" % depot_path )
2305         return ""
2306
2307 class P4Sync(Command, P4UserMap):
2308     delete_actions = ( "delete", "move/delete", "purge" )
2309
2310     def __init__(self):
2311         Command.__init__(self)
2312         P4UserMap.__init__(self)
2313         self.options = [
2314                 optparse.make_option("--branch", dest="branch"),
2315                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2316                 optparse.make_option("--changesfile", dest="changesFile"),
2317                 optparse.make_option("--silent", dest="silent", action="store_true"),
2318                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2319                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2320                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2321                                      help="Import into refs/heads/ , not refs/remotes"),
2322                 optparse.make_option("--max-changes", dest="maxChanges",
2323                                      help="Maximum number of changes to import"),
2324                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2325                                      help="Internal block size to use when iteratively calling p4 changes"),
2326                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2327                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2328                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2329                                      help="Only sync files that are included in the Perforce Client Spec"),
2330                 optparse.make_option("-/", dest="cloneExclude",
2331                                      action="append", type="string",
2332                                      help="exclude depot path"),
2333         ]
2334         self.description = """Imports from Perforce into a git repository.\n
2335     example:
2336     //depot/my/project/ -- to import the current head
2337     //depot/my/project/@all -- to import everything
2338     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2339
2340     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2341
2342         self.usage += " //depot/path[@revRange]"
2343         self.silent = False
2344         self.createdBranches = set()
2345         self.committedChanges = set()
2346         self.branch = ""
2347         self.detectBranches = False
2348         self.detectLabels = False
2349         self.importLabels = False
2350         self.changesFile = ""
2351         self.syncWithOrigin = True
2352         self.importIntoRemotes = True
2353         self.maxChanges = ""
2354         self.changes_block_size = None
2355         self.keepRepoPath = False
2356         self.depotPaths = None
2357         self.p4BranchesInGit = []
2358         self.cloneExclude = []
2359         self.useClientSpec = False
2360         self.useClientSpec_from_options = False
2361         self.clientSpecDirs = None
2362         self.tempBranches = []
2363         self.tempBranchLocation = "refs/git-p4-tmp"
2364         self.largeFileSystem = None
2365
2366         if gitConfig('git-p4.largeFileSystem'):
2367             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2368             self.largeFileSystem = largeFileSystemConstructor(
2369                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2370             )
2371
2372         if gitConfig("git-p4.syncFromOrigin") == "false":
2373             self.syncWithOrigin = False
2374
2375     # This is required for the "append" cloneExclude action
2376     def ensure_value(self, attr, value):
2377         if not hasattr(self, attr) or getattr(self, attr) is None:
2378             setattr(self, attr, value)
2379         return getattr(self, attr)
2380
2381     # Force a checkpoint in fast-import and wait for it to finish
2382     def checkpoint(self):
2383         self.gitStream.write("checkpoint\n\n")
2384         self.gitStream.write("progress checkpoint\n\n")
2385         out = self.gitOutput.readline()
2386         if self.verbose:
2387             print "checkpoint finished: " + out
2388
2389     def extractFilesFromCommit(self, commit):
2390         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2391                              for path in self.cloneExclude]
2392         files = []
2393         fnum = 0
2394         while commit.has_key("depotFile%s" % fnum):
2395             path =  commit["depotFile%s" % fnum]
2396
2397             if [p for p in self.cloneExclude
2398                 if p4PathStartsWith(path, p)]:
2399                 found = False
2400             else:
2401                 found = [p for p in self.depotPaths
2402                          if p4PathStartsWith(path, p)]
2403             if not found:
2404                 fnum = fnum + 1
2405                 continue
2406
2407             file = {}
2408             file["path"] = path
2409             file["rev"] = commit["rev%s" % fnum]
2410             file["action"] = commit["action%s" % fnum]
2411             file["type"] = commit["type%s" % fnum]
2412             files.append(file)
2413             fnum = fnum + 1
2414         return files
2415
2416     def extractJobsFromCommit(self, commit):
2417         jobs = []
2418         jnum = 0
2419         while commit.has_key("job%s" % jnum):
2420             job = commit["job%s" % jnum]
2421             jobs.append(job)
2422             jnum = jnum + 1
2423         return jobs
2424
2425     def stripRepoPath(self, path, prefixes):
2426         """When streaming files, this is called to map a p4 depot path
2427            to where it should go in git.  The prefixes are either
2428            self.depotPaths, or self.branchPrefixes in the case of
2429            branch detection."""
2430
2431         if self.useClientSpec:
2432             # branch detection moves files up a level (the branch name)
2433             # from what client spec interpretation gives
2434             path = self.clientSpecDirs.map_in_client(path)
2435             if self.detectBranches:
2436                 for b in self.knownBranches:
2437                     if path.startswith(b + "/"):
2438                         path = path[len(b)+1:]
2439
2440         elif self.keepRepoPath:
2441             # Preserve everything in relative path name except leading
2442             # //depot/; just look at first prefix as they all should
2443             # be in the same depot.
2444             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2445             if p4PathStartsWith(path, depot):
2446                 path = path[len(depot):]
2447
2448         else:
2449             for p in prefixes:
2450                 if p4PathStartsWith(path, p):
2451                     path = path[len(p):]
2452                     break
2453
2454         path = wildcard_decode(path)
2455         return path
2456
2457     def splitFilesIntoBranches(self, commit):
2458         """Look at each depotFile in the commit to figure out to what
2459            branch it belongs."""
2460
2461         if self.clientSpecDirs:
2462             files = self.extractFilesFromCommit(commit)
2463             self.clientSpecDirs.update_client_spec_path_cache(files)
2464
2465         branches = {}
2466         fnum = 0
2467         while commit.has_key("depotFile%s" % fnum):
2468             path =  commit["depotFile%s" % fnum]
2469             found = [p for p in self.depotPaths
2470                      if p4PathStartsWith(path, p)]
2471             if not found:
2472                 fnum = fnum + 1
2473                 continue
2474
2475             file = {}
2476             file["path"] = path
2477             file["rev"] = commit["rev%s" % fnum]
2478             file["action"] = commit["action%s" % fnum]
2479             file["type"] = commit["type%s" % fnum]
2480             fnum = fnum + 1
2481
2482             # start with the full relative path where this file would
2483             # go in a p4 client
2484             if self.useClientSpec:
2485                 relPath = self.clientSpecDirs.map_in_client(path)
2486             else:
2487                 relPath = self.stripRepoPath(path, self.depotPaths)
2488
2489             for branch in self.knownBranches.keys():
2490                 # add a trailing slash so that a commit into qt/4.2foo
2491                 # doesn't end up in qt/4.2, e.g.
2492                 if relPath.startswith(branch + "/"):
2493                     if branch not in branches:
2494                         branches[branch] = []
2495                     branches[branch].append(file)
2496                     break
2497
2498         return branches
2499
2500     def writeToGitStream(self, gitMode, relPath, contents):
2501         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2502         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2503         for d in contents:
2504             self.gitStream.write(d)
2505         self.gitStream.write('\n')
2506
2507     def encodeWithUTF8(self, path):
2508         try:
2509             path.decode('ascii')
2510         except:
2511             encoding = 'utf8'
2512             if gitConfig('git-p4.pathEncoding'):
2513                 encoding = gitConfig('git-p4.pathEncoding')
2514             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2515             if self.verbose:
2516                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2517         return path
2518
2519     # output one file from the P4 stream
2520     # - helper for streamP4Files
2521
2522     def streamOneP4File(self, file, contents):
2523         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2524         relPath = self.encodeWithUTF8(relPath)
2525         if verbose:
2526             size = int(self.stream_file['fileSize'])
2527             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2528             sys.stdout.flush()
2529
2530         (type_base, type_mods) = split_p4_type(file["type"])
2531
2532         git_mode = "100644"
2533         if "x" in type_mods:
2534             git_mode = "100755"
2535         if type_base == "symlink":
2536             git_mode = "120000"
2537             # p4 print on a symlink sometimes contains "target\n";
2538             # if it does, remove the newline
2539             data = ''.join(contents)
2540             if not data:
2541                 # Some version of p4 allowed creating a symlink that pointed
2542                 # to nothing.  This causes p4 errors when checking out such
2543                 # a change, and errors here too.  Work around it by ignoring
2544                 # the bad symlink; hopefully a future change fixes it.
2545                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2546                 return
2547             elif data[-1] == '\n':
2548                 contents = [data[:-1]]
2549             else:
2550                 contents = [data]
2551
2552         if type_base == "utf16":
2553             # p4 delivers different text in the python output to -G
2554             # than it does when using "print -o", or normal p4 client
2555             # operations.  utf16 is converted to ascii or utf8, perhaps.
2556             # But ascii text saved as -t utf16 is completely mangled.
2557             # Invoke print -o to get the real contents.
2558             #
2559             # On windows, the newlines will always be mangled by print, so put
2560             # them back too.  This is not needed to the cygwin windows version,
2561             # just the native "NT" type.
2562             #
2563             try:
2564                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2565             except Exception as e:
2566                 if 'Translation of file content failed' in str(e):
2567                     type_base = 'binary'
2568                 else:
2569                     raise e
2570             else:
2571                 if p4_version_string().find('/NT') >= 0:
2572                     text = text.replace('\r\n', '\n')
2573                 contents = [ text ]
2574
2575         if type_base == "apple":
2576             # Apple filetype files will be streamed as a concatenation of
2577             # its appledouble header and the contents.  This is useless
2578             # on both macs and non-macs.  If using "print -q -o xx", it
2579             # will create "xx" with the data, and "%xx" with the header.
2580             # This is also not very useful.
2581             #
2582             # Ideally, someday, this script can learn how to generate
2583             # appledouble files directly and import those to git, but
2584             # non-mac machines can never find a use for apple filetype.
2585             print "\nIgnoring apple filetype file %s" % file['depotFile']
2586             return
2587
2588         # Note that we do not try to de-mangle keywords on utf16 files,
2589         # even though in theory somebody may want that.
2590         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2591         if pattern:
2592             regexp = re.compile(pattern, re.VERBOSE)
2593             text = ''.join(contents)
2594             text = regexp.sub(r'$\1$', text)
2595             contents = [ text ]
2596
2597         if self.largeFileSystem:
2598             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2599
2600         self.writeToGitStream(git_mode, relPath, contents)
2601
2602     def streamOneP4Deletion(self, file):
2603         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2604         relPath = self.encodeWithUTF8(relPath)
2605         if verbose:
2606             sys.stdout.write("delete %s\n" % relPath)
2607             sys.stdout.flush()
2608         self.gitStream.write("D %s\n" % relPath)
2609
2610         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2611             self.largeFileSystem.removeLargeFile(relPath)
2612
2613     # handle another chunk of streaming data
2614     def streamP4FilesCb(self, marshalled):
2615
2616         # catch p4 errors and complain
2617         err = None
2618         if "code" in marshalled:
2619             if marshalled["code"] == "error":
2620                 if "data" in marshalled:
2621                     err = marshalled["data"].rstrip()
2622
2623         if not err and 'fileSize' in self.stream_file:
2624             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2625             if required_bytes > 0:
2626                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2627                     os.getcwd(), required_bytes/1024/1024
2628                 )
2629
2630         if err:
2631             f = None
2632             if self.stream_have_file_info:
2633                 if "depotFile" in self.stream_file:
2634                     f = self.stream_file["depotFile"]
2635             # force a failure in fast-import, else an empty
2636             # commit will be made
2637             self.gitStream.write("\n")
2638             self.gitStream.write("die-now\n")
2639             self.gitStream.close()
2640             # ignore errors, but make sure it exits first
2641             self.importProcess.wait()
2642             if f:
2643                 die("Error from p4 print for %s: %s" % (f, err))
2644             else:
2645                 die("Error from p4 print: %s" % err)
2646
2647         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2648             # start of a new file - output the old one first
2649             self.streamOneP4File(self.stream_file, self.stream_contents)
2650             self.stream_file = {}
2651             self.stream_contents = []
2652             self.stream_have_file_info = False
2653
2654         # pick up the new file information... for the
2655         # 'data' field we need to append to our array
2656         for k in marshalled.keys():
2657             if k == 'data':
2658                 if 'streamContentSize' not in self.stream_file:
2659                     self.stream_file['streamContentSize'] = 0
2660                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2661                 self.stream_contents.append(marshalled['data'])
2662             else:
2663                 self.stream_file[k] = marshalled[k]
2664
2665         if (verbose and
2666             'streamContentSize' in self.stream_file and
2667             'fileSize' in self.stream_file and
2668             'depotFile' in self.stream_file):
2669             size = int(self.stream_file["fileSize"])
2670             if size > 0:
2671                 progress = 100*self.stream_file['streamContentSize']/size
2672                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2673                 sys.stdout.flush()
2674
2675         self.stream_have_file_info = True
2676
2677     # Stream directly from "p4 files" into "git fast-import"
2678     def streamP4Files(self, files):
2679         filesForCommit = []
2680         filesToRead = []
2681         filesToDelete = []
2682
2683         for f in files:
2684             filesForCommit.append(f)
2685             if f['action'] in self.delete_actions:
2686                 filesToDelete.append(f)
2687             else:
2688                 filesToRead.append(f)
2689
2690         # deleted files...
2691         for f in filesToDelete:
2692             self.streamOneP4Deletion(f)
2693
2694         if len(filesToRead) > 0:
2695             self.stream_file = {}
2696             self.stream_contents = []
2697             self.stream_have_file_info = False
2698
2699             # curry self argument
2700             def streamP4FilesCbSelf(entry):
2701                 self.streamP4FilesCb(entry)
2702
2703             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2704
2705             p4CmdList(["-x", "-", "print"],
2706                       stdin=fileArgs,
2707                       cb=streamP4FilesCbSelf)
2708
2709             # do the last chunk
2710             if self.stream_file.has_key('depotFile'):
2711                 self.streamOneP4File(self.stream_file, self.stream_contents)
2712
2713     def make_email(self, userid):
2714         if userid in self.users:
2715             return self.users[userid]
2716         else:
2717             return "%s <a@b>" % userid
2718
2719     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2720         """ Stream a p4 tag.
2721         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2722         """
2723
2724         if verbose:
2725             print "writing tag %s for commit %s" % (labelName, commit)
2726         gitStream.write("tag %s\n" % labelName)
2727         gitStream.write("from %s\n" % commit)
2728
2729         if labelDetails.has_key('Owner'):
2730             owner = labelDetails["Owner"]
2731         else:
2732             owner = None
2733
2734         # Try to use the owner of the p4 label, or failing that,
2735         # the current p4 user id.
2736         if owner:
2737             email = self.make_email(owner)
2738         else:
2739             email = self.make_email(self.p4UserId())
2740         tagger = "%s %s %s" % (email, epoch, self.tz)
2741
2742         gitStream.write("tagger %s\n" % tagger)
2743
2744         print "labelDetails=",labelDetails
2745         if labelDetails.has_key('Description'):
2746             description = labelDetails['Description']
2747         else:
2748             description = 'Label from git p4'
2749
2750         gitStream.write("data %d\n" % len(description))
2751         gitStream.write(description)
2752         gitStream.write("\n")
2753
2754     def inClientSpec(self, path):
2755         if not self.clientSpecDirs:
2756             return True
2757         inClientSpec = self.clientSpecDirs.map_in_client(path)
2758         if not inClientSpec and self.verbose:
2759             print('Ignoring file outside of client spec: {0}'.format(path))
2760         return inClientSpec
2761
2762     def hasBranchPrefix(self, path):
2763         if not self.branchPrefixes:
2764             return True
2765         hasPrefix = [p for p in self.branchPrefixes
2766                         if p4PathStartsWith(path, p)]
2767         if not hasPrefix and self.verbose:
2768             print('Ignoring file outside of prefix: {0}'.format(path))
2769         return hasPrefix
2770
2771     def commit(self, details, files, branch, parent = ""):
2772         epoch = details["time"]
2773         author = details["user"]
2774         jobs = self.extractJobsFromCommit(details)
2775
2776         if self.verbose:
2777             print('commit into {0}'.format(branch))
2778
2779         if self.clientSpecDirs:
2780             self.clientSpecDirs.update_client_spec_path_cache(files)
2781
2782         files = [f for f in files
2783             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2784
2785         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2786             print('Ignoring revision {0} as it would produce an empty commit.'
2787                 .format(details['change']))
2788             return
2789
2790         self.gitStream.write("commit %s\n" % branch)
2791         self.gitStream.write("mark :%s\n" % details["change"])
2792         self.committedChanges.add(int(details["change"]))
2793         committer = ""
2794         if author not in self.users:
2795             self.getUserMapFromPerforceServer()
2796         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2797
2798         self.gitStream.write("committer %s\n" % committer)
2799
2800         self.gitStream.write("data <<EOT\n")
2801         self.gitStream.write(details["desc"])
2802         if len(jobs) > 0:
2803             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2804         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2805                              (','.join(self.branchPrefixes), details["change"]))
2806         if len(details['options']) > 0:
2807             self.gitStream.write(": options = %s" % details['options'])
2808         self.gitStream.write("]\nEOT\n\n")
2809
2810         if len(parent) > 0:
2811             if self.verbose:
2812                 print "parent %s" % parent
2813             self.gitStream.write("from %s\n" % parent)
2814
2815         self.streamP4Files(files)
2816         self.gitStream.write("\n")
2817
2818         change = int(details["change"])
2819
2820         if self.labels.has_key(change):
2821             label = self.labels[change]
2822             labelDetails = label[0]
2823             labelRevisions = label[1]
2824             if self.verbose:
2825                 print "Change %s is labelled %s" % (change, labelDetails)
2826
2827             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2828                                                 for p in self.branchPrefixes])
2829
2830             if len(files) == len(labelRevisions):
2831
2832                 cleanedFiles = {}
2833                 for info in files:
2834                     if info["action"] in self.delete_actions:
2835                         continue
2836                     cleanedFiles[info["depotFile"]] = info["rev"]
2837
2838                 if cleanedFiles == labelRevisions:
2839                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2840
2841                 else:
2842                     if not self.silent:
2843                         print ("Tag %s does not match with change %s: files do not match."
2844                                % (labelDetails["label"], change))
2845
2846             else:
2847                 if not self.silent:
2848                     print ("Tag %s does not match with change %s: file count is different."
2849                            % (labelDetails["label"], change))
2850
2851     # Build a dictionary of changelists and labels, for "detect-labels" option.
2852     def getLabels(self):
2853         self.labels = {}
2854
2855         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2856         if len(l) > 0 and not self.silent:
2857             print "Finding files belonging to labels in %s" % `self.depotPaths`
2858
2859         for output in l:
2860             label = output["label"]
2861             revisions = {}
2862             newestChange = 0
2863             if self.verbose:
2864                 print "Querying files for label %s" % label
2865             for file in p4CmdList(["files"] +
2866                                       ["%s...@%s" % (p, label)
2867                                           for p in self.depotPaths]):
2868                 revisions[file["depotFile"]] = file["rev"]
2869                 change = int(file["change"])
2870                 if change > newestChange:
2871                     newestChange = change
2872
2873             self.labels[newestChange] = [output, revisions]
2874
2875         if self.verbose:
2876             print "Label changes: %s" % self.labels.keys()
2877
2878     # Import p4 labels as git tags. A direct mapping does not
2879     # exist, so assume that if all the files are at the same revision
2880     # then we can use that, or it's something more complicated we should
2881     # just ignore.
2882     def importP4Labels(self, stream, p4Labels):
2883         if verbose:
2884             print "import p4 labels: " + ' '.join(p4Labels)
2885
2886         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2887         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2888         if len(validLabelRegexp) == 0:
2889             validLabelRegexp = defaultLabelRegexp
2890         m = re.compile(validLabelRegexp)
2891
2892         for name in p4Labels:
2893             commitFound = False
2894
2895             if not m.match(name):
2896                 if verbose:
2897                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2898                 continue
2899
2900             if name in ignoredP4Labels:
2901                 continue
2902
2903             labelDetails = p4CmdList(['label', "-o", name])[0]
2904
2905             # get the most recent changelist for each file in this label
2906             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2907                                 for p in self.depotPaths])
2908
2909             if change.has_key('change'):
2910                 # find the corresponding git commit; take the oldest commit
2911                 changelist = int(change['change'])
2912                 if changelist in self.committedChanges:
2913                     gitCommit = ":%d" % changelist       # use a fast-import mark
2914                     commitFound = True
2915                 else:
2916                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2917                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2918                     if len(gitCommit) == 0:
2919                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2920                     else:
2921                         commitFound = True
2922                         gitCommit = gitCommit.strip()
2923
2924                 if commitFound:
2925                     # Convert from p4 time format
2926                     try:
2927                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2928                     except ValueError:
2929                         print "Could not convert label time %s" % labelDetails['Update']
2930                         tmwhen = 1
2931
2932                     when = int(time.mktime(tmwhen))
2933                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2934                     if verbose:
2935                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2936             else:
2937                 if verbose:
2938                     print "Label %s has no changelists - possibly deleted?" % name
2939
2940             if not commitFound:
2941                 # We can't import this label; don't try again as it will get very
2942                 # expensive repeatedly fetching all the files for labels that will
2943                 # never be imported. If the label is moved in the future, the
2944                 # ignore will need to be removed manually.
2945                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2946
2947     def guessProjectName(self):
2948         for p in self.depotPaths:
2949             if p.endswith("/"):
2950                 p = p[:-1]
2951             p = p[p.strip().rfind("/") + 1:]
2952             if not p.endswith("/"):
2953                p += "/"
2954             return p
2955
2956     def getBranchMapping(self):
2957         lostAndFoundBranches = set()
2958
2959         user = gitConfig("git-p4.branchUser")
2960         if len(user) > 0:
2961             command = "branches -u %s" % user
2962         else:
2963             command = "branches"
2964
2965         for info in p4CmdList(command):
2966             details = p4Cmd(["branch", "-o", info["branch"]])
2967             viewIdx = 0
2968             while details.has_key("View%s" % viewIdx):
2969                 paths = details["View%s" % viewIdx].split(" ")
2970                 viewIdx = viewIdx + 1
2971                 # require standard //depot/foo/... //depot/bar/... mapping
2972                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2973                     continue
2974                 source = paths[0]
2975                 destination = paths[1]
2976                 ## HACK
2977                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2978                     source = source[len(self.depotPaths[0]):-4]
2979                     destination = destination[len(self.depotPaths[0]):-4]
2980
2981                     if destination in self.knownBranches:
2982                         if not self.silent:
2983                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2984                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2985                         continue
2986
2987                     self.knownBranches[destination] = source
2988
2989                     lostAndFoundBranches.discard(destination)
2990
2991                     if source not in self.knownBranches:
2992                         lostAndFoundBranches.add(source)
2993
2994         # Perforce does not strictly require branches to be defined, so we also
2995         # check git config for a branch list.
2996         #
2997         # Example of branch definition in git config file:
2998         # [git-p4]
2999         #   branchList=main:branchA
3000         #   branchList=main:branchB
3001         #   branchList=branchA:branchC
3002         configBranches = gitConfigList("git-p4.branchList")
3003         for branch in configBranches:
3004             if branch:
3005                 (source, destination) = branch.split(":")
3006                 self.knownBranches[destination] = source
3007
3008                 lostAndFoundBranches.discard(destination)
3009
3010                 if source not in self.knownBranches:
3011                     lostAndFoundBranches.add(source)
3012
3013
3014         for branch in lostAndFoundBranches:
3015             self.knownBranches[branch] = branch
3016
3017     def getBranchMappingFromGitBranches(self):
3018         branches = p4BranchesInGit(self.importIntoRemotes)
3019         for branch in branches.keys():
3020             if branch == "master":
3021                 branch = "main"
3022             else:
3023                 branch = branch[len(self.projectName):]
3024             self.knownBranches[branch] = branch
3025
3026     def updateOptionDict(self, d):
3027         option_keys = {}
3028         if self.keepRepoPath:
3029             option_keys['keepRepoPath'] = 1
3030
3031         d["options"] = ' '.join(sorted(option_keys.keys()))
3032
3033     def readOptions(self, d):
3034         self.keepRepoPath = (d.has_key('options')
3035                              and ('keepRepoPath' in d['options']))
3036
3037     def gitRefForBranch(self, branch):
3038         if branch == "main":
3039             return self.refPrefix + "master"
3040
3041         if len(branch) <= 0:
3042             return branch
3043
3044         return self.refPrefix + self.projectName + branch
3045
3046     def gitCommitByP4Change(self, ref, change):
3047         if self.verbose:
3048             print "looking in ref " + ref + " for change %s using bisect..." % change
3049
3050         earliestCommit = ""
3051         latestCommit = parseRevision(ref)
3052
3053         while True:
3054             if self.verbose:
3055                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3056             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3057             if len(next) == 0:
3058                 if self.verbose:
3059                     print "argh"
3060                 return ""
3061             log = extractLogMessageFromGitCommit(next)
3062             settings = extractSettingsGitLog(log)
3063             currentChange = int(settings['change'])
3064             if self.verbose:
3065                 print "current change %s" % currentChange
3066
3067             if currentChange == change:
3068                 if self.verbose:
3069                     print "found %s" % next
3070                 return next
3071
3072             if currentChange < change:
3073                 earliestCommit = "^%s" % next
3074             else:
3075                 latestCommit = "%s" % next
3076
3077         return ""
3078
3079     def importNewBranch(self, branch, maxChange):
3080         # make fast-import flush all changes to disk and update the refs using the checkpoint
3081         # command so that we can try to find the branch parent in the git history
3082         self.gitStream.write("checkpoint\n\n");
3083         self.gitStream.flush();
3084         branchPrefix = self.depotPaths[0] + branch + "/"
3085         range = "@1,%s" % maxChange
3086         #print "prefix" + branchPrefix
3087         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3088         if len(changes) <= 0:
3089             return False
3090         firstChange = changes[0]
3091         #print "first change in branch: %s" % firstChange
3092         sourceBranch = self.knownBranches[branch]
3093         sourceDepotPath = self.depotPaths[0] + sourceBranch
3094         sourceRef = self.gitRefForBranch(sourceBranch)
3095         #print "source " + sourceBranch
3096
3097         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3098         #print "branch parent: %s" % branchParentChange
3099         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3100         if len(gitParent) > 0:
3101             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3102             #print "parent git commit: %s" % gitParent
3103
3104         self.importChanges(changes)
3105         return True
3106
3107     def searchParent(self, parent, branch, target):
3108         parentFound = False
3109         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3110                                      "--no-merges", parent]):
3111             blob = blob.strip()
3112             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3113                 parentFound = True
3114                 if self.verbose:
3115                     print "Found parent of %s in commit %s" % (branch, blob)
3116                 break
3117         if parentFound:
3118             return blob
3119         else:
3120             return None
3121
3122     def importChanges(self, changes):
3123         cnt = 1
3124         for change in changes:
3125             description = p4_describe(change)
3126             self.updateOptionDict(description)
3127
3128             if not self.silent:
3129                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3130                 sys.stdout.flush()
3131             cnt = cnt + 1
3132
3133             try:
3134                 if self.detectBranches:
3135                     branches = self.splitFilesIntoBranches(description)
3136                     for branch in branches.keys():
3137                         ## HACK  --hwn
3138                         branchPrefix = self.depotPaths[0] + branch + "/"
3139                         self.branchPrefixes = [ branchPrefix ]
3140
3141                         parent = ""
3142
3143                         filesForCommit = branches[branch]
3144
3145                         if self.verbose:
3146                             print "branch is %s" % branch
3147
3148                         self.updatedBranches.add(branch)
3149
3150                         if branch not in self.createdBranches:
3151                             self.createdBranches.add(branch)
3152                             parent = self.knownBranches[branch]
3153                             if parent == branch:
3154                                 parent = ""
3155                             else:
3156                                 fullBranch = self.projectName + branch
3157                                 if fullBranch not in self.p4BranchesInGit:
3158                                     if not self.silent:
3159                                         print("\n    Importing new branch %s" % fullBranch);
3160                                     if self.importNewBranch(branch, change - 1):
3161                                         parent = ""
3162                                         self.p4BranchesInGit.append(fullBranch)
3163                                     if not self.silent:
3164                                         print("\n    Resuming with change %s" % change);
3165
3166                                 if self.verbose:
3167                                     print "parent determined through known branches: %s" % parent
3168
3169                         branch = self.gitRefForBranch(branch)
3170                         parent = self.gitRefForBranch(parent)
3171
3172                         if self.verbose:
3173                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3174
3175                         if len(parent) == 0 and branch in self.initialParents:
3176                             parent = self.initialParents[branch]
3177                             del self.initialParents[branch]
3178
3179                         blob = None
3180                         if len(parent) > 0:
3181                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3182                             if self.verbose:
3183                                 print "Creating temporary branch: " + tempBranch
3184                             self.commit(description, filesForCommit, tempBranch)
3185                             self.tempBranches.append(tempBranch)
3186                             self.checkpoint()
3187                             blob = self.searchParent(parent, branch, tempBranch)
3188                         if blob:
3189                             self.commit(description, filesForCommit, branch, blob)
3190                         else:
3191                             if self.verbose:
3192                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3193                             self.commit(description, filesForCommit, branch, parent)
3194                 else:
3195                     files = self.extractFilesFromCommit(description)
3196                     self.commit(description, files, self.branch,
3197                                 self.initialParent)
3198                     # only needed once, to connect to the previous commit
3199                     self.initialParent = ""
3200             except IOError:
3201                 print self.gitError.read()
3202                 sys.exit(1)
3203
3204     def importHeadRevision(self, revision):
3205         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3206
3207         details = {}
3208         details["user"] = "git perforce import user"
3209         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3210                            % (' '.join(self.depotPaths), revision))
3211         details["change"] = revision
3212         newestRevision = 0
3213
3214         fileCnt = 0
3215         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3216
3217         for info in p4CmdList(["files"] + fileArgs):
3218
3219             if 'code' in info and info['code'] == 'error':
3220                 sys.stderr.write("p4 returned an error: %s\n"
3221                                  % info['data'])
3222                 if info['data'].find("must refer to client") >= 0:
3223                     sys.stderr.write("This particular p4 error is misleading.\n")
3224                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3225                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3226                 sys.exit(1)
3227             if 'p4ExitCode' in info:
3228                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3229                 sys.exit(1)
3230
3231
3232             change = int(info["change"])
3233             if change > newestRevision:
3234                 newestRevision = change
3235
3236             if info["action"] in self.delete_actions:
3237                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3238                 #fileCnt = fileCnt + 1
3239                 continue
3240
3241             for prop in ["depotFile", "rev", "action", "type" ]:
3242                 details["%s%s" % (prop, fileCnt)] = info[prop]
3243
3244             fileCnt = fileCnt + 1
3245
3246         details["change"] = newestRevision
3247
3248         # Use time from top-most change so that all git p4 clones of
3249         # the same p4 repo have the same commit SHA1s.
3250         res = p4_describe(newestRevision)
3251         details["time"] = res["time"]
3252
3253         self.updateOptionDict(details)
3254         try:
3255             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3256         except IOError:
3257             print "IO error with git fast-import. Is your git version recent enough?"
3258             print self.gitError.read()
3259
3260
3261     def run(self, args):
3262         self.depotPaths = []
3263         self.changeRange = ""
3264         self.previousDepotPaths = []
3265         self.hasOrigin = False
3266
3267         # map from branch depot path to parent branch
3268         self.knownBranches = {}
3269         self.initialParents = {}
3270
3271         if self.importIntoRemotes:
3272             self.refPrefix = "refs/remotes/p4/"
3273         else:
3274             self.refPrefix = "refs/heads/p4/"
3275
3276         if self.syncWithOrigin:
3277             self.hasOrigin = originP4BranchesExist()
3278             if self.hasOrigin:
3279                 if not self.silent:
3280                     print 'Syncing with origin first, using "git fetch origin"'
3281                 system("git fetch origin")
3282
3283         branch_arg_given = bool(self.branch)
3284         if len(self.branch) == 0:
3285             self.branch = self.refPrefix + "master"
3286             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3287                 system("git update-ref %s refs/heads/p4" % self.branch)
3288                 system("git branch -D p4")
3289
3290         # accept either the command-line option, or the configuration variable
3291         if self.useClientSpec:
3292             # will use this after clone to set the variable
3293             self.useClientSpec_from_options = True
3294         else:
3295             if gitConfigBool("git-p4.useclientspec"):
3296                 self.useClientSpec = True
3297         if self.useClientSpec:
3298             self.clientSpecDirs = getClientSpec()
3299
3300         # TODO: should always look at previous commits,
3301         # merge with previous imports, if possible.
3302         if args == []:
3303             if self.hasOrigin:
3304                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3305
3306             # branches holds mapping from branch name to sha1
3307             branches = p4BranchesInGit(self.importIntoRemotes)
3308
3309             # restrict to just this one, disabling detect-branches
3310             if branch_arg_given:
3311                 short = self.branch.split("/")[-1]
3312                 if short in branches:
3313                     self.p4BranchesInGit = [ short ]
3314             else:
3315                 self.p4BranchesInGit = branches.keys()
3316
3317             if len(self.p4BranchesInGit) > 1:
3318                 if not self.silent:
3319                     print "Importing from/into multiple branches"
3320                 self.detectBranches = True
3321                 for branch in branches.keys():
3322                     self.initialParents[self.refPrefix + branch] = \
3323                         branches[branch]
3324
3325             if self.verbose:
3326                 print "branches: %s" % self.p4BranchesInGit
3327
3328             p4Change = 0
3329             for branch in self.p4BranchesInGit:
3330                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3331
3332                 settings = extractSettingsGitLog(logMsg)
3333
3334                 self.readOptions(settings)
3335                 if (settings.has_key('depot-paths')
3336                     and settings.has_key ('change')):
3337                     change = int(settings['change']) + 1
3338                     p4Change = max(p4Change, change)
3339
3340                     depotPaths = sorted(settings['depot-paths'])
3341                     if self.previousDepotPaths == []:
3342                         self.previousDepotPaths = depotPaths
3343                     else:
3344                         paths = []
3345                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3346                             prev_list = prev.split("/")
3347                             cur_list = cur.split("/")
3348                             for i in range(0, min(len(cur_list), len(prev_list))):
3349                                 if cur_list[i] <> prev_list[i]:
3350                                     i = i - 1
3351                                     break
3352
3353                             paths.append ("/".join(cur_list[:i + 1]))
3354
3355                         self.previousDepotPaths = paths
3356
3357             if p4Change > 0:
3358                 self.depotPaths = sorted(self.previousDepotPaths)
3359                 self.changeRange = "@%s,#head" % p4Change
3360                 if not self.silent and not self.detectBranches:
3361                     print "Performing incremental import into %s git branch" % self.branch
3362
3363         # accept multiple ref name abbreviations:
3364         #    refs/foo/bar/branch -> use it exactly
3365         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3366         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3367         if not self.branch.startswith("refs/"):
3368             if self.importIntoRemotes:
3369                 prepend = "refs/remotes/"
3370             else:
3371                 prepend = "refs/heads/"
3372             if not self.branch.startswith("p4/"):
3373                 prepend += "p4/"
3374             self.branch = prepend + self.branch
3375
3376         if len(args) == 0 and self.depotPaths:
3377             if not self.silent:
3378                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3379         else:
3380             if self.depotPaths and self.depotPaths != args:
3381                 print ("previous import used depot path %s and now %s was specified. "
3382                        "This doesn't work!" % (' '.join (self.depotPaths),
3383                                                ' '.join (args)))
3384                 sys.exit(1)
3385
3386             self.depotPaths = sorted(args)
3387
3388         revision = ""
3389         self.users = {}
3390
3391         # Make sure no revision specifiers are used when --changesfile
3392         # is specified.
3393         bad_changesfile = False
3394         if len(self.changesFile) > 0:
3395             for p in self.depotPaths:
3396                 if p.find("@") >= 0 or p.find("#") >= 0:
3397                     bad_changesfile = True
3398                     break
3399         if bad_changesfile:
3400             die("Option --changesfile is incompatible with revision specifiers")
3401
3402         newPaths = []
3403         for p in self.depotPaths:
3404             if p.find("@") != -1:
3405                 atIdx = p.index("@")
3406                 self.changeRange = p[atIdx:]
3407                 if self.changeRange == "@all":
3408                     self.changeRange = ""
3409                 elif ',' not in self.changeRange:
3410                     revision = self.changeRange
3411                     self.changeRange = ""
3412                 p = p[:atIdx]
3413             elif p.find("#") != -1:
3414                 hashIdx = p.index("#")
3415                 revision = p[hashIdx:]
3416                 p = p[:hashIdx]
3417             elif self.previousDepotPaths == []:
3418                 # pay attention to changesfile, if given, else import
3419                 # the entire p4 tree at the head revision
3420                 if len(self.changesFile) == 0:
3421                     revision = "#head"
3422
3423             p = re.sub ("\.\.\.$", "", p)
3424             if not p.endswith("/"):
3425                 p += "/"
3426
3427             newPaths.append(p)
3428
3429         self.depotPaths = newPaths
3430
3431         # --detect-branches may change this for each branch
3432         self.branchPrefixes = self.depotPaths
3433
3434         self.loadUserMapFromCache()
3435         self.labels = {}
3436         if self.detectLabels:
3437             self.getLabels();
3438
3439         if self.detectBranches:
3440             ## FIXME - what's a P4 projectName ?
3441             self.projectName = self.guessProjectName()
3442
3443             if self.hasOrigin:
3444                 self.getBranchMappingFromGitBranches()
3445             else:
3446                 self.getBranchMapping()
3447             if self.verbose:
3448                 print "p4-git branches: %s" % self.p4BranchesInGit
3449                 print "initial parents: %s" % self.initialParents
3450             for b in self.p4BranchesInGit:
3451                 if b != "master":
3452
3453                     ## FIXME
3454                     b = b[len(self.projectName):]
3455                 self.createdBranches.add(b)
3456
3457         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3458
3459         self.importProcess = subprocess.Popen(["git", "fast-import"],
3460                                               stdin=subprocess.PIPE,
3461                                               stdout=subprocess.PIPE,
3462                                               stderr=subprocess.PIPE);
3463         self.gitOutput = self.importProcess.stdout
3464         self.gitStream = self.importProcess.stdin
3465         self.gitError = self.importProcess.stderr
3466
3467         if revision:
3468             self.importHeadRevision(revision)
3469         else:
3470             changes = []
3471
3472             if len(self.changesFile) > 0:
3473                 output = open(self.changesFile).readlines()
3474                 changeSet = set()
3475                 for line in output:
3476                     changeSet.add(int(line))
3477
3478                 for change in changeSet:
3479                     changes.append(change)
3480
3481                 changes.sort()
3482             else:
3483                 # catch "git p4 sync" with no new branches, in a repo that
3484                 # does not have any existing p4 branches
3485                 if len(args) == 0:
3486                     if not self.p4BranchesInGit:
3487                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3488
3489                     # The default branch is master, unless --branch is used to
3490                     # specify something else.  Make sure it exists, or complain
3491                     # nicely about how to use --branch.
3492                     if not self.detectBranches:
3493                         if not branch_exists(self.branch):
3494                             if branch_arg_given:
3495                                 die("Error: branch %s does not exist." % self.branch)
3496                             else:
3497                                 die("Error: no branch %s; perhaps specify one with --branch." %
3498                                     self.branch)
3499
3500                 if self.verbose:
3501                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3502                                                               self.changeRange)
3503                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3504
3505                 if len(self.maxChanges) > 0:
3506                     changes = changes[:min(int(self.maxChanges), len(changes))]
3507
3508             if len(changes) == 0:
3509                 if not self.silent:
3510                     print "No changes to import!"
3511             else:
3512                 if not self.silent and not self.detectBranches:
3513                     print "Import destination: %s" % self.branch
3514
3515                 self.updatedBranches = set()
3516
3517                 if not self.detectBranches:
3518                     if args:
3519                         # start a new branch
3520                         self.initialParent = ""
3521                     else:
3522                         # build on a previous revision
3523                         self.initialParent = parseRevision(self.branch)
3524
3525                 self.importChanges(changes)
3526
3527                 if not self.silent:
3528                     print ""
3529                     if len(self.updatedBranches) > 0:
3530                         sys.stdout.write("Updated branches: ")
3531                         for b in self.updatedBranches:
3532                             sys.stdout.write("%s " % b)
3533                         sys.stdout.write("\n")
3534
3535         if gitConfigBool("git-p4.importLabels"):
3536             self.importLabels = True
3537
3538         if self.importLabels:
3539             p4Labels = getP4Labels(self.depotPaths)
3540             gitTags = getGitTags()
3541
3542             missingP4Labels = p4Labels - gitTags
3543             self.importP4Labels(self.gitStream, missingP4Labels)
3544
3545         self.gitStream.close()
3546         if self.importProcess.wait() != 0:
3547             die("fast-import failed: %s" % self.gitError.read())
3548         self.gitOutput.close()
3549         self.gitError.close()
3550
3551         # Cleanup temporary branches created during import
3552         if self.tempBranches != []:
3553             for branch in self.tempBranches:
3554                 read_pipe("git update-ref -d %s" % branch)
3555             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3556
3557         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3558         # a convenient shortcut refname "p4".
3559         if self.importIntoRemotes:
3560             head_ref = self.refPrefix + "HEAD"
3561             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3562                 system(["git", "symbolic-ref", head_ref, self.branch])
3563
3564         return True
3565
3566 class P4Rebase(Command):
3567     def __init__(self):
3568         Command.__init__(self)
3569         self.options = [
3570                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3571         ]
3572         self.importLabels = False
3573         self.description = ("Fetches the latest revision from perforce and "
3574                             + "rebases the current work (branch) against it")
3575
3576     def run(self, args):
3577         sync = P4Sync()
3578         sync.importLabels = self.importLabels
3579         sync.run([])
3580
3581         return self.rebase()
3582
3583     def rebase(self):
3584         if os.system("git update-index --refresh") != 0:
3585             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3586         if len(read_pipe("git diff-index HEAD --")) > 0:
3587             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3588
3589         [upstream, settings] = findUpstreamBranchPoint()
3590         if len(upstream) == 0:
3591             die("Cannot find upstream branchpoint for rebase")
3592
3593         # the branchpoint may be p4/foo~3, so strip off the parent
3594         upstream = re.sub("~[0-9]+$", "", upstream)
3595
3596         print "Rebasing the current branch onto %s" % upstream
3597         oldHead = read_pipe("git rev-parse HEAD").strip()
3598         system("git rebase %s" % upstream)
3599         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3600         return True
3601
3602 class P4Clone(P4Sync):
3603     def __init__(self):
3604         P4Sync.__init__(self)
3605         self.description = "Creates a new git repository and imports from Perforce into it"
3606         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3607         self.options += [
3608             optparse.make_option("--destination", dest="cloneDestination",
3609                                  action='store', default=None,
3610                                  help="where to leave result of the clone"),
3611             optparse.make_option("--bare", dest="cloneBare",
3612                                  action="store_true", default=False),
3613         ]
3614         self.cloneDestination = None
3615         self.needsGit = False
3616         self.cloneBare = False
3617
3618     def defaultDestination(self, args):
3619         ## TODO: use common prefix of args?
3620         depotPath = args[0]
3621         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3622         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3623         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3624         depotDir = re.sub(r"/$", "", depotDir)
3625         return os.path.split(depotDir)[1]
3626
3627     def run(self, args):
3628         if len(args) < 1:
3629             return False
3630
3631         if self.keepRepoPath and not self.cloneDestination:
3632             sys.stderr.write("Must specify destination for --keep-path\n")
3633             sys.exit(1)
3634
3635         depotPaths = args
3636
3637         if not self.cloneDestination and len(depotPaths) > 1:
3638             self.cloneDestination = depotPaths[-1]
3639             depotPaths = depotPaths[:-1]
3640
3641         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3642         for p in depotPaths:
3643             if not p.startswith("//"):
3644                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3645                 return False
3646
3647         if not self.cloneDestination:
3648             self.cloneDestination = self.defaultDestination(args)
3649
3650         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3651
3652         if not os.path.exists(self.cloneDestination):
3653             os.makedirs(self.cloneDestination)
3654         chdir(self.cloneDestination)
3655
3656         init_cmd = [ "git", "init" ]
3657         if self.cloneBare:
3658             init_cmd.append("--bare")
3659         retcode = subprocess.call(init_cmd)
3660         if retcode:
3661             raise CalledProcessError(retcode, init_cmd)
3662
3663         if not P4Sync.run(self, depotPaths):
3664             return False
3665
3666         # create a master branch and check out a work tree
3667         if gitBranchExists(self.branch):
3668             system([ "git", "branch", "master", self.branch ])
3669             if not self.cloneBare:
3670                 system([ "git", "checkout", "-f" ])
3671         else:
3672             print 'Not checking out any branch, use ' \
3673                   '"git checkout -q -b master <branch>"'
3674
3675         # auto-set this variable if invoked with --use-client-spec
3676         if self.useClientSpec_from_options:
3677             system("git config --bool git-p4.useclientspec true")
3678
3679         return True
3680
3681 class P4Branches(Command):
3682     def __init__(self):
3683         Command.__init__(self)
3684         self.options = [ ]
3685         self.description = ("Shows the git branches that hold imports and their "
3686                             + "corresponding perforce depot paths")
3687         self.verbose = False
3688
3689     def run(self, args):
3690         if originP4BranchesExist():
3691             createOrUpdateBranchesFromOrigin()
3692
3693         cmdline = "git rev-parse --symbolic "
3694         cmdline += " --remotes"
3695
3696         for line in read_pipe_lines(cmdline):
3697             line = line.strip()
3698
3699             if not line.startswith('p4/') or line == "p4/HEAD":
3700                 continue
3701             branch = line
3702
3703             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3704             settings = extractSettingsGitLog(log)
3705
3706             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3707         return True
3708
3709 class HelpFormatter(optparse.IndentedHelpFormatter):
3710     def __init__(self):
3711         optparse.IndentedHelpFormatter.__init__(self)
3712
3713     def format_description(self, description):
3714         if description:
3715             return description + "\n"
3716         else:
3717             return ""
3718
3719 def printUsage(commands):
3720     print "usage: %s <command> [options]" % sys.argv[0]
3721     print ""
3722     print "valid commands: %s" % ", ".join(commands)
3723     print ""
3724     print "Try %s <command> --help for command specific help." % sys.argv[0]
3725     print ""
3726
3727 commands = {
3728     "debug" : P4Debug,
3729     "submit" : P4Submit,
3730     "commit" : P4Submit,
3731     "sync" : P4Sync,
3732     "rebase" : P4Rebase,
3733     "clone" : P4Clone,
3734     "rollback" : P4RollBack,
3735     "branches" : P4Branches
3736 }
3737
3738
3739 def main():
3740     if len(sys.argv[1:]) == 0:
3741         printUsage(commands.keys())
3742         sys.exit(2)
3743
3744     cmdName = sys.argv[1]
3745     try:
3746         klass = commands[cmdName]
3747         cmd = klass()
3748     except KeyError:
3749         print "unknown command %s" % cmdName
3750         print ""
3751         printUsage(commands.keys())
3752         sys.exit(2)
3753
3754     options = cmd.options
3755     cmd.gitdir = os.environ.get("GIT_DIR", None)
3756
3757     args = sys.argv[2:]
3758
3759     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3760     if cmd.needsGit:
3761         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3762
3763     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3764                                    options,
3765                                    description = cmd.description,
3766                                    formatter = HelpFormatter())
3767
3768     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3769     global verbose
3770     verbose = cmd.verbose
3771     if cmd.needsGit:
3772         if cmd.gitdir == None:
3773             cmd.gitdir = os.path.abspath(".git")
3774             if not isValidGitDir(cmd.gitdir):
3775                 # "rev-parse --git-dir" without arguments will try $PWD/.git
3776                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3777                 if os.path.exists(cmd.gitdir):
3778                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3779                     if len(cdup) > 0:
3780                         chdir(cdup);
3781
3782         if not isValidGitDir(cmd.gitdir):
3783             if isValidGitDir(cmd.gitdir + "/.git"):
3784                 cmd.gitdir += "/.git"
3785             else:
3786                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3787
3788         # so git commands invoked from the P4 workspace will succeed
3789         os.environ["GIT_DIR"] = cmd.gitdir
3790
3791     if not cmd.run(args):
3792         parser.print_help()
3793         sys.exit(2)
3794
3795
3796 if __name__ == '__main__':
3797     main()