serialize collection of changed submodules
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28
29 try:
30     from subprocess import CalledProcessError
31 except ImportError:
32     # from python2.7:subprocess.py
33     # Exception classes used by this module.
34     class CalledProcessError(Exception):
35         """This exception is raised when a process run by check_call() returns
36         a non-zero exit status.  The exit status will be stored in the
37         returncode attribute."""
38         def __init__(self, returncode, cmd):
39             self.returncode = returncode
40             self.cmd = cmd
41         def __str__(self):
42             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
44 verbose = False
45
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
48
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize = 512
51
52 def p4_build_cmd(cmd):
53     """Build a suitable p4 command line.
54
55     This consolidates building and returning a p4 command line into one
56     location. It means that hooking into the environment, or other configuration
57     can be done more easily.
58     """
59     real_cmd = ["p4"]
60
61     user = gitConfig("git-p4.user")
62     if len(user) > 0:
63         real_cmd += ["-u",user]
64
65     password = gitConfig("git-p4.password")
66     if len(password) > 0:
67         real_cmd += ["-P", password]
68
69     port = gitConfig("git-p4.port")
70     if len(port) > 0:
71         real_cmd += ["-p", port]
72
73     host = gitConfig("git-p4.host")
74     if len(host) > 0:
75         real_cmd += ["-H", host]
76
77     client = gitConfig("git-p4.client")
78     if len(client) > 0:
79         real_cmd += ["-c", client]
80
81
82     if isinstance(cmd,basestring):
83         real_cmd = ' '.join(real_cmd) + ' ' + cmd
84     else:
85         real_cmd += cmd
86     return real_cmd
87
88 def chdir(path, is_client_path=False):
89     """Do chdir to the given path, and set the PWD environment
90        variable for use by P4.  It does not look at getcwd() output.
91        Since we're not using the shell, it is necessary to set the
92        PWD environment variable explicitly.
93
94        Normally, expand the path to force it to be absolute.  This
95        addresses the use of relative path names inside P4 settings,
96        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
97        as given; it looks for .p4config using PWD.
98
99        If is_client_path, the path was handed to us directly by p4,
100        and may be a symbolic link.  Do not call os.getcwd() in this
101        case, because it will cause p4 to think that PWD is not inside
102        the client path.
103        """
104
105     os.chdir(path)
106     if not is_client_path:
107         path = os.getcwd()
108     os.environ['PWD'] = path
109
110 def calcDiskFree():
111     """Return free space in bytes on the disk of the given dirname."""
112     if platform.system() == 'Windows':
113         free_bytes = ctypes.c_ulonglong(0)
114         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115         return free_bytes.value
116     else:
117         st = os.statvfs(os.getcwd())
118         return st.f_bavail * st.f_frsize
119
120 def die(msg):
121     if verbose:
122         raise Exception(msg)
123     else:
124         sys.stderr.write(msg + "\n")
125         sys.exit(1)
126
127 def write_pipe(c, stdin):
128     if verbose:
129         sys.stderr.write('Writing pipe: %s\n' % str(c))
130
131     expand = isinstance(c,basestring)
132     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133     pipe = p.stdin
134     val = pipe.write(stdin)
135     pipe.close()
136     if p.wait():
137         die('Command failed: %s' % str(c))
138
139     return val
140
141 def p4_write_pipe(c, stdin):
142     real_cmd = p4_build_cmd(c)
143     return write_pipe(real_cmd, stdin)
144
145 def read_pipe(c, ignore_error=False):
146     if verbose:
147         sys.stderr.write('Reading pipe: %s\n' % str(c))
148
149     expand = isinstance(c,basestring)
150     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151     (out, err) = p.communicate()
152     if p.returncode != 0 and not ignore_error:
153         die('Command failed: %s\nError: %s' % (str(c), err))
154     return out
155
156 def p4_read_pipe(c, ignore_error=False):
157     real_cmd = p4_build_cmd(c)
158     return read_pipe(real_cmd, ignore_error)
159
160 def read_pipe_lines(c):
161     if verbose:
162         sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164     expand = isinstance(c, basestring)
165     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166     pipe = p.stdout
167     val = pipe.readlines()
168     if pipe.close() or p.wait():
169         die('Command failed: %s' % str(c))
170
171     return val
172
173 def p4_read_pipe_lines(c):
174     """Specifically invoke p4 on the command supplied. """
175     real_cmd = p4_build_cmd(c)
176     return read_pipe_lines(real_cmd)
177
178 def p4_has_command(cmd):
179     """Ask p4 for help on this command.  If it returns an error, the
180        command does not exist in this version of p4."""
181     real_cmd = p4_build_cmd(["help", cmd])
182     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183                                    stderr=subprocess.PIPE)
184     p.communicate()
185     return p.returncode == 0
186
187 def p4_has_move_command():
188     """See if the move command exists, that it supports -k, and that
189        it has not been administratively disabled.  The arguments
190        must be correct, but the filenames do not have to exist.  Use
191        ones with wildcards so even if they exist, it will fail."""
192
193     if not p4_has_command("move"):
194         return False
195     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197     (out, err) = p.communicate()
198     # return code will be 1 in either case
199     if err.find("Invalid option") >= 0:
200         return False
201     if err.find("disabled") >= 0:
202         return False
203     # assume it failed because @... was invalid changelist
204     return True
205
206 def system(cmd, ignore_error=False):
207     expand = isinstance(cmd,basestring)
208     if verbose:
209         sys.stderr.write("executing %s\n" % str(cmd))
210     retcode = subprocess.call(cmd, shell=expand)
211     if retcode and not ignore_error:
212         raise CalledProcessError(retcode, cmd)
213
214     return retcode
215
216 def p4_system(cmd):
217     """Specifically invoke p4 as the system command. """
218     real_cmd = p4_build_cmd(cmd)
219     expand = isinstance(real_cmd, basestring)
220     retcode = subprocess.call(real_cmd, shell=expand)
221     if retcode:
222         raise CalledProcessError(retcode, real_cmd)
223
224 _p4_version_string = None
225 def p4_version_string():
226     """Read the version string, showing just the last line, which
227        hopefully is the interesting version bit.
228
229        $ p4 -V
230        Perforce - The Fast Software Configuration Management System.
231        Copyright 1995-2011 Perforce Software.  All rights reserved.
232        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233     """
234     global _p4_version_string
235     if not _p4_version_string:
236         a = p4_read_pipe_lines(["-V"])
237         _p4_version_string = a[-1].rstrip()
238     return _p4_version_string
239
240 def p4_integrate(src, dest):
241     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
242
243 def p4_sync(f, *options):
244     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
245
246 def p4_add(f):
247     # forcibly add file names with wildcards
248     if wildcard_present(f):
249         p4_system(["add", "-f", f])
250     else:
251         p4_system(["add", f])
252
253 def p4_delete(f):
254     p4_system(["delete", wildcard_encode(f)])
255
256 def p4_edit(f, *options):
257     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
258
259 def p4_revert(f):
260     p4_system(["revert", wildcard_encode(f)])
261
262 def p4_reopen(type, f):
263     p4_system(["reopen", "-t", type, wildcard_encode(f)])
264
265 def p4_move(src, dest):
266     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
267
268 def p4_last_change():
269     results = p4CmdList(["changes", "-m", "1"])
270     return int(results[0]['change'])
271
272 def p4_describe(change):
273     """Make sure it returns a valid result by checking for
274        the presence of field "time".  Return a dict of the
275        results."""
276
277     ds = p4CmdList(["describe", "-s", str(change)])
278     if len(ds) != 1:
279         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280
281     d = ds[0]
282
283     if "p4ExitCode" in d:
284         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
285                                                       str(d)))
286     if "code" in d:
287         if d["code"] == "error":
288             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
289
290     if "time" not in d:
291         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
292
293     return d
294
295 #
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers.  See "p4 help filetypes"
298 # for a list and explanation.
299 #
300 def split_p4_type(p4type):
301
302     p4_filetypes_historical = {
303         "ctempobj": "binary+Sw",
304         "ctext": "text+C",
305         "cxtext": "text+Cx",
306         "ktext": "text+k",
307         "kxtext": "text+kx",
308         "ltext": "text+F",
309         "tempobj": "binary+FSw",
310         "ubinary": "binary+F",
311         "uresource": "resource+F",
312         "uxbinary": "binary+Fx",
313         "xbinary": "binary+x",
314         "xltext": "text+Fx",
315         "xtempobj": "binary+Swx",
316         "xtext": "text+x",
317         "xunicode": "unicode+x",
318         "xutf16": "utf16+x",
319     }
320     if p4type in p4_filetypes_historical:
321         p4type = p4_filetypes_historical[p4type]
322     mods = ""
323     s = p4type.split("+")
324     base = s[0]
325     mods = ""
326     if len(s) > 1:
327         mods = s[1]
328     return (base, mods)
329
330 #
331 # return the raw p4 type of a file (text, text+ko, etc)
332 #
333 def p4_type(f):
334     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
335     return results[0]['headType']
336
337 #
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
340 #
341 def p4_keywords_regexp_for_type(base, type_mods):
342     if base in ("text", "unicode", "binary"):
343         kwords = None
344         if "ko" in type_mods:
345             kwords = 'Id|Header'
346         elif "k" in type_mods:
347             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348         else:
349             return None
350         pattern = r"""
351             \$              # Starts with a dollar, followed by...
352             (%s)            # one of the keywords, followed by...
353             (:[^$\n]+)?     # possibly an old expansion, followed by...
354             \$              # another dollar
355             """ % kwords
356         return pattern
357     else:
358         return None
359
360 #
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
364 #
365 def p4_keywords_regexp_for_file(file):
366     if not os.path.exists(file):
367         return None
368     else:
369         (type_base, type_mods) = split_p4_type(p4_type(file))
370         return p4_keywords_regexp_for_type(type_base, type_mods)
371
372 def setP4ExecBit(file, mode):
373     # Reopens an already open file and changes the execute bit to match
374     # the execute bit setting in the passed in mode.
375
376     p4Type = "+x"
377
378     if not isModeExec(mode):
379         p4Type = getP4OpenedType(file)
380         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382         if p4Type[-1] == "+":
383             p4Type = p4Type[0:-1]
384
385     p4_reopen(p4Type, file)
386
387 def getP4OpenedType(file):
388     # Returns the perforce file type for the given file.
389
390     result = p4_read_pipe(["opened", wildcard_encode(file)])
391     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
392     if match:
393         return match.group(1)
394     else:
395         die("Could not determine file type for %s (result: '%s')" % (file, result))
396
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths):
399     labels = set()
400     if isinstance(depotPaths,basestring):
401         depotPaths = [depotPaths]
402
403     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
404         label = l['label']
405         labels.add(label)
406
407     return labels
408
409 # Return the set of all git tags
410 def getGitTags():
411     gitTags = set()
412     for line in read_pipe_lines(["git", "tag"]):
413         tag = line.strip()
414         gitTags.add(tag)
415     return gitTags
416
417 def diffTreePattern():
418     # This is a simple generator for the diff tree regex pattern. This could be
419     # a class variable if this and parseDiffTreeEntry were a part of a class.
420     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421     while True:
422         yield pattern
423
424 def parseDiffTreeEntry(entry):
425     """Parses a single diff tree entry into its component elements.
426
427     See git-diff-tree(1) manpage for details about the format of the diff
428     output. This method returns a dictionary with the following elements:
429
430     src_mode - The mode of the source file
431     dst_mode - The mode of the destination file
432     src_sha1 - The sha1 for the source file
433     dst_sha1 - The sha1 fr the destination file
434     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435     status_score - The score for the status (applicable for 'C' and 'R'
436                    statuses). This is None if there is no score.
437     src - The path for the source file.
438     dst - The path for the destination file. This is only present for
439           copy or renames. If it is not present, this is None.
440
441     If the pattern is not matched, None is returned."""
442
443     match = diffTreePattern().next().match(entry)
444     if match:
445         return {
446             'src_mode': match.group(1),
447             'dst_mode': match.group(2),
448             'src_sha1': match.group(3),
449             'dst_sha1': match.group(4),
450             'status': match.group(5),
451             'status_score': match.group(6),
452             'src': match.group(7),
453             'dst': match.group(10)
454         }
455     return None
456
457 def isModeExec(mode):
458     # Returns True if the given git mode represents an executable file,
459     # otherwise False.
460     return mode[-3:] == "755"
461
462 def isModeExecChanged(src_mode, dst_mode):
463     return isModeExec(src_mode) != isModeExec(dst_mode)
464
465 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
466
467     if isinstance(cmd,basestring):
468         cmd = "-G " + cmd
469         expand = True
470     else:
471         cmd = ["-G"] + cmd
472         expand = False
473
474     cmd = p4_build_cmd(cmd)
475     if verbose:
476         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
477
478     # Use a temporary file to avoid deadlocks without
479     # subprocess.communicate(), which would put another copy
480     # of stdout into memory.
481     stdin_file = None
482     if stdin is not None:
483         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
484         if isinstance(stdin,basestring):
485             stdin_file.write(stdin)
486         else:
487             for i in stdin:
488                 stdin_file.write(i + '\n')
489         stdin_file.flush()
490         stdin_file.seek(0)
491
492     p4 = subprocess.Popen(cmd,
493                           shell=expand,
494                           stdin=stdin_file,
495                           stdout=subprocess.PIPE)
496
497     result = []
498     try:
499         while True:
500             entry = marshal.load(p4.stdout)
501             if cb is not None:
502                 cb(entry)
503             else:
504                 result.append(entry)
505     except EOFError:
506         pass
507     exitCode = p4.wait()
508     if exitCode != 0:
509         entry = {}
510         entry["p4ExitCode"] = exitCode
511         result.append(entry)
512
513     return result
514
515 def p4Cmd(cmd):
516     list = p4CmdList(cmd)
517     result = {}
518     for entry in list:
519         result.update(entry)
520     return result;
521
522 def p4Where(depotPath):
523     if not depotPath.endswith("/"):
524         depotPath += "/"
525     depotPathLong = depotPath + "..."
526     outputList = p4CmdList(["where", depotPathLong])
527     output = None
528     for entry in outputList:
529         if "depotFile" in entry:
530             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531             # The base path always ends with "/...".
532             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
533                 output = entry
534                 break
535         elif "data" in entry:
536             data = entry.get("data")
537             space = data.find(" ")
538             if data[:space] == depotPath:
539                 output = entry
540                 break
541     if output == None:
542         return ""
543     if output["code"] == "error":
544         return ""
545     clientPath = ""
546     if "path" in output:
547         clientPath = output.get("path")
548     elif "data" in output:
549         data = output.get("data")
550         lastSpace = data.rfind(" ")
551         clientPath = data[lastSpace + 1:]
552
553     if clientPath.endswith("..."):
554         clientPath = clientPath[:-3]
555     return clientPath
556
557 def currentGitBranch():
558     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
559     if retcode != 0:
560         # on a detached head
561         return None
562     else:
563         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
564
565 def isValidGitDir(path):
566     if (os.path.exists(path + "/HEAD")
567         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
568         return True;
569     return False
570
571 def parseRevision(ref):
572     return read_pipe("git rev-parse %s" % ref).strip()
573
574 def branchExists(ref):
575     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
576                      ignore_error=True)
577     return len(rev) > 0
578
579 def extractLogMessageFromGitCommit(commit):
580     logMessage = ""
581
582     ## fixme: title is first line of commit, not 1st paragraph.
583     foundTitle = False
584     for log in read_pipe_lines("git cat-file commit %s" % commit):
585        if not foundTitle:
586            if len(log) == 1:
587                foundTitle = True
588            continue
589
590        logMessage += log
591     return logMessage
592
593 def extractSettingsGitLog(log):
594     values = {}
595     for line in log.split("\n"):
596         line = line.strip()
597         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
598         if not m:
599             continue
600
601         assignments = m.group(1).split (':')
602         for a in assignments:
603             vals = a.split ('=')
604             key = vals[0].strip()
605             val = ('='.join (vals[1:])).strip()
606             if val.endswith ('\"') and val.startswith('"'):
607                 val = val[1:-1]
608
609             values[key] = val
610
611     paths = values.get("depot-paths")
612     if not paths:
613         paths = values.get("depot-path")
614     if paths:
615         values['depot-paths'] = paths.split(',')
616     return values
617
618 def gitBranchExists(branch):
619     proc = subprocess.Popen(["git", "rev-parse", branch],
620                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
621     return proc.wait() == 0;
622
623 _gitConfig = {}
624
625 def gitConfig(key, typeSpecifier=None):
626     if not _gitConfig.has_key(key):
627         cmd = [ "git", "config" ]
628         if typeSpecifier:
629             cmd += [ typeSpecifier ]
630         cmd += [ key ]
631         s = read_pipe(cmd, ignore_error=True)
632         _gitConfig[key] = s.strip()
633     return _gitConfig[key]
634
635 def gitConfigBool(key):
636     """Return a bool, using git config --bool.  It is True only if the
637        variable is set to true, and False if set to false or not present
638        in the config."""
639
640     if not _gitConfig.has_key(key):
641         _gitConfig[key] = gitConfig(key, '--bool') == "true"
642     return _gitConfig[key]
643
644 def gitConfigInt(key):
645     if not _gitConfig.has_key(key):
646         cmd = [ "git", "config", "--int", key ]
647         s = read_pipe(cmd, ignore_error=True)
648         v = s.strip()
649         try:
650             _gitConfig[key] = int(gitConfig(key, '--int'))
651         except ValueError:
652             _gitConfig[key] = None
653     return _gitConfig[key]
654
655 def gitConfigList(key):
656     if not _gitConfig.has_key(key):
657         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658         _gitConfig[key] = s.strip().split(os.linesep)
659         if _gitConfig[key] == ['']:
660             _gitConfig[key] = []
661     return _gitConfig[key]
662
663 def p4BranchesInGit(branchesAreInRemotes=True):
664     """Find all the branches whose names start with "p4/", looking
665        in remotes or heads as specified by the argument.  Return
666        a dictionary of { branch: revision } for each one found.
667        The branch names are the short names, without any
668        "p4/" prefix."""
669
670     branches = {}
671
672     cmdline = "git rev-parse --symbolic "
673     if branchesAreInRemotes:
674         cmdline += "--remotes"
675     else:
676         cmdline += "--branches"
677
678     for line in read_pipe_lines(cmdline):
679         line = line.strip()
680
681         # only import to p4/
682         if not line.startswith('p4/'):
683             continue
684         # special symbolic ref to p4/master
685         if line == "p4/HEAD":
686             continue
687
688         # strip off p4/ prefix
689         branch = line[len("p4/"):]
690
691         branches[branch] = parseRevision(line)
692
693     return branches
694
695 def branch_exists(branch):
696     """Make sure that the given ref name really exists."""
697
698     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700     out, _ = p.communicate()
701     if p.returncode:
702         return False
703     # expect exactly one line of output: the branch name
704     return out.rstrip() == branch
705
706 def findUpstreamBranchPoint(head = "HEAD"):
707     branches = p4BranchesInGit()
708     # map from depot-path to branch name
709     branchByDepotPath = {}
710     for branch in branches.keys():
711         tip = branches[branch]
712         log = extractLogMessageFromGitCommit(tip)
713         settings = extractSettingsGitLog(log)
714         if settings.has_key("depot-paths"):
715             paths = ",".join(settings["depot-paths"])
716             branchByDepotPath[paths] = "remotes/p4/" + branch
717
718     settings = None
719     parent = 0
720     while parent < 65535:
721         commit = head + "~%s" % parent
722         log = extractLogMessageFromGitCommit(commit)
723         settings = extractSettingsGitLog(log)
724         if settings.has_key("depot-paths"):
725             paths = ",".join(settings["depot-paths"])
726             if branchByDepotPath.has_key(paths):
727                 return [branchByDepotPath[paths], settings]
728
729         parent = parent + 1
730
731     return ["", settings]
732
733 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
734     if not silent:
735         print ("Creating/updating branch(es) in %s based on origin branch(es)"
736                % localRefPrefix)
737
738     originPrefix = "origin/p4/"
739
740     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
741         line = line.strip()
742         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
743             continue
744
745         headName = line[len(originPrefix):]
746         remoteHead = localRefPrefix + headName
747         originHead = line
748
749         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750         if (not original.has_key('depot-paths')
751             or not original.has_key('change')):
752             continue
753
754         update = False
755         if not gitBranchExists(remoteHead):
756             if verbose:
757                 print "creating %s" % remoteHead
758             update = True
759         else:
760             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761             if settings.has_key('change') > 0:
762                 if settings['depot-paths'] == original['depot-paths']:
763                     originP4Change = int(original['change'])
764                     p4Change = int(settings['change'])
765                     if originP4Change > p4Change:
766                         print ("%s (%s) is newer than %s (%s). "
767                                "Updating p4 branch from origin."
768                                % (originHead, originP4Change,
769                                   remoteHead, p4Change))
770                         update = True
771                 else:
772                     print ("Ignoring: %s was imported from %s while "
773                            "%s was imported from %s"
774                            % (originHead, ','.join(original['depot-paths']),
775                               remoteHead, ','.join(settings['depot-paths'])))
776
777         if update:
778             system("git update-ref %s %s" % (remoteHead, originHead))
779
780 def originP4BranchesExist():
781         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
782
783
784 def p4ParseNumericChangeRange(parts):
785     changeStart = int(parts[0][1:])
786     if parts[1] == '#head':
787         changeEnd = p4_last_change()
788     else:
789         changeEnd = int(parts[1])
790
791     return (changeStart, changeEnd)
792
793 def chooseBlockSize(blockSize):
794     if blockSize:
795         return blockSize
796     else:
797         return defaultBlockSize
798
799 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
800     assert depotPaths
801
802     # Parse the change range into start and end. Try to find integer
803     # revision ranges as these can be broken up into blocks to avoid
804     # hitting server-side limits (maxrows, maxscanresults). But if
805     # that doesn't work, fall back to using the raw revision specifier
806     # strings, without using block mode.
807
808     if changeRange is None or changeRange == '':
809         changeStart = 1
810         changeEnd = p4_last_change()
811         block_size = chooseBlockSize(requestedBlockSize)
812     else:
813         parts = changeRange.split(',')
814         assert len(parts) == 2
815         try:
816             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817             block_size = chooseBlockSize(requestedBlockSize)
818         except:
819             changeStart = parts[0][1:]
820             changeEnd = parts[1]
821             if requestedBlockSize:
822                 die("cannot use --changes-block-size with non-numeric revisions")
823             block_size = None
824
825     changes = []
826
827     # Retrieve changes a block at a time, to prevent running
828     # into a MaxResults/MaxScanRows error from the server.
829
830     while True:
831         cmd = ['changes']
832
833         if block_size:
834             end = min(changeEnd, changeStart + block_size)
835             revisionRange = "%d,%d" % (changeStart, end)
836         else:
837             revisionRange = "%s,%s" % (changeStart, changeEnd)
838
839         for p in depotPaths:
840             cmd += ["%s...@%s" % (p, revisionRange)]
841
842         # Insert changes in chronological order
843         for line in reversed(p4_read_pipe_lines(cmd)):
844             changes.append(int(line.split(" ")[1]))
845
846         if not block_size:
847             break
848
849         if end >= changeEnd:
850             break
851
852         changeStart = end + 1
853
854     changes = sorted(changes)
855     return changes
856
857 def p4PathStartsWith(path, prefix):
858     # This method tries to remedy a potential mixed-case issue:
859     #
860     # If UserA adds  //depot/DirA/file1
861     # and UserB adds //depot/dira/file2
862     #
863     # we may or may not have a problem. If you have core.ignorecase=true,
864     # we treat DirA and dira as the same directory
865     if gitConfigBool("core.ignorecase"):
866         return path.lower().startswith(prefix.lower())
867     return path.startswith(prefix)
868
869 def getClientSpec():
870     """Look at the p4 client spec, create a View() object that contains
871        all the mappings, and return it."""
872
873     specList = p4CmdList("client -o")
874     if len(specList) != 1:
875         die('Output from "client -o" is %d lines, expecting 1' %
876             len(specList))
877
878     # dictionary of all client parameters
879     entry = specList[0]
880
881     # the //client/ name
882     client_name = entry["Client"]
883
884     # just the keys that start with "View"
885     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
886
887     # hold this new View
888     view = View(client_name)
889
890     # append the lines, in order, to the view
891     for view_num in range(len(view_keys)):
892         k = "View%d" % view_num
893         if k not in view_keys:
894             die("Expected view key %s missing" % k)
895         view.append(entry[k])
896
897     return view
898
899 def getClientRoot():
900     """Grab the client directory."""
901
902     output = p4CmdList("client -o")
903     if len(output) != 1:
904         die('Output from "client -o" is %d lines, expecting 1' % len(output))
905
906     entry = output[0]
907     if "Root" not in entry:
908         die('Client has no "Root"')
909
910     return entry["Root"]
911
912 #
913 # P4 wildcards are not allowed in filenames.  P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
916 #
917 def wildcard_decode(path):
918     # Search for and fix just these four characters.  Do % last so
919     # that fixing it does not inadvertently create new %-escapes.
920     # Cannot have * in a filename in windows; untested as to
921     # what p4 would do in such a case.
922     if not platform.system() == "Windows":
923         path = path.replace("%2A", "*")
924     path = path.replace("%23", "#") \
925                .replace("%40", "@") \
926                .replace("%25", "%")
927     return path
928
929 def wildcard_encode(path):
930     # do % first to avoid double-encoding the %s introduced here
931     path = path.replace("%", "%25") \
932                .replace("*", "%2A") \
933                .replace("#", "%23") \
934                .replace("@", "%40")
935     return path
936
937 def wildcard_present(path):
938     m = re.search("[*#@%]", path)
939     return m is not None
940
941 class LargeFileSystem(object):
942     """Base class for large file system support."""
943
944     def __init__(self, writeToGitStream):
945         self.largeFiles = set()
946         self.writeToGitStream = writeToGitStream
947
948     def generatePointer(self, cloneDestination, contentFile):
949         """Return the content of a pointer file that is stored in Git instead of
950            the actual content."""
951         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
952
953     def pushFile(self, localLargeFile):
954         """Push the actual content which is not stored in the Git repository to
955            a server."""
956         assert False, "Method 'pushFile' required in " + self.__class__.__name__
957
958     def hasLargeFileExtension(self, relPath):
959         return reduce(
960             lambda a, b: a or b,
961             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
962             False
963         )
964
965     def generateTempFile(self, contents):
966         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
967         for d in contents:
968             contentFile.write(d)
969         contentFile.close()
970         return contentFile.name
971
972     def exceedsLargeFileThreshold(self, relPath, contents):
973         if gitConfigInt('git-p4.largeFileThreshold'):
974             contentsSize = sum(len(d) for d in contents)
975             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
976                 return True
977         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978             contentsSize = sum(len(d) for d in contents)
979             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
980                 return False
981             contentTempFile = self.generateTempFile(contents)
982             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
985             zf.close()
986             compressedContentsSize = zf.infolist()[0].compress_size
987             os.remove(contentTempFile)
988             os.remove(compressedContentFile.name)
989             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
990                 return True
991         return False
992
993     def addLargeFile(self, relPath):
994         self.largeFiles.add(relPath)
995
996     def removeLargeFile(self, relPath):
997         self.largeFiles.remove(relPath)
998
999     def isLargeFile(self, relPath):
1000         return relPath in self.largeFiles
1001
1002     def processContent(self, git_mode, relPath, contents):
1003         """Processes the content of git fast import. This method decides if a
1004            file is stored in the large file system and handles all necessary
1005            steps."""
1006         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007             contentTempFile = self.generateTempFile(contents)
1008             (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1009
1010             # Move temp file to final location in large file system
1011             largeFileDir = os.path.dirname(localLargeFile)
1012             if not os.path.isdir(largeFileDir):
1013                 os.makedirs(largeFileDir)
1014             shutil.move(contentTempFile, localLargeFile)
1015             self.addLargeFile(relPath)
1016             if gitConfigBool('git-p4.largeFilePush'):
1017                 self.pushFile(localLargeFile)
1018             if verbose:
1019                 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020         return (git_mode, contents)
1021
1022 class MockLFS(LargeFileSystem):
1023     """Mock large file system for testing."""
1024
1025     def generatePointer(self, contentFile):
1026         """The pointer content is the original content prefixed with "pointer-".
1027            The local filename of the large file storage is derived from the file content.
1028            """
1029         with open(contentFile, 'r') as f:
1030             content = next(f)
1031             gitMode = '100644'
1032             pointerContents = 'pointer-' + content
1033             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034             return (gitMode, pointerContents, localLargeFile)
1035
1036     def pushFile(self, localLargeFile):
1037         """The remote filename of the large file storage is the same as the local
1038            one but in a different directory.
1039            """
1040         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041         if not os.path.exists(remotePath):
1042             os.makedirs(remotePath)
1043         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1044
1045 class GitLFS(LargeFileSystem):
1046     """Git LFS as backend for the git-p4 large file system.
1047        See https://git-lfs.github.com/ for details."""
1048
1049     def __init__(self, *args):
1050         LargeFileSystem.__init__(self, *args)
1051         self.baseGitAttributes = []
1052
1053     def generatePointer(self, contentFile):
1054         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055            mode and content which is stored in the Git repository instead of
1056            the actual content. Return also the new location of the actual
1057            content.
1058            """
1059         pointerProcess = subprocess.Popen(
1060             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061             stdout=subprocess.PIPE
1062         )
1063         pointerFile = pointerProcess.stdout.read()
1064         if pointerProcess.wait():
1065             os.remove(contentFile)
1066             die('git-lfs pointer command failed. Did you install the extension?')
1067
1068         # Git LFS removed the preamble in the output of the 'pointer' command
1069         # starting from version 1.2.0. Check for the preamble here to support
1070         # earlier versions.
1071         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1072         if pointerFile.startswith('Git LFS pointer for'):
1073             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1074
1075         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1076         localLargeFile = os.path.join(
1077             os.getcwd(),
1078             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1079             oid,
1080         )
1081         # LFS Spec states that pointer files should not have the executable bit set.
1082         gitMode = '100644'
1083         return (gitMode, pointerFile, localLargeFile)
1084
1085     def pushFile(self, localLargeFile):
1086         uploadProcess = subprocess.Popen(
1087             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1088         )
1089         if uploadProcess.wait():
1090             die('git-lfs push command failed. Did you define a remote?')
1091
1092     def generateGitAttributes(self):
1093         return (
1094             self.baseGitAttributes +
1095             [
1096                 '\n',
1097                 '#\n',
1098                 '# Git LFS (see https://git-lfs.github.com/)\n',
1099                 '#\n',
1100             ] +
1101             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1102                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1103             ] +
1104             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1105                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1106             ]
1107         )
1108
1109     def addLargeFile(self, relPath):
1110         LargeFileSystem.addLargeFile(self, relPath)
1111         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1112
1113     def removeLargeFile(self, relPath):
1114         LargeFileSystem.removeLargeFile(self, relPath)
1115         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1116
1117     def processContent(self, git_mode, relPath, contents):
1118         if relPath == '.gitattributes':
1119             self.baseGitAttributes = contents
1120             return (git_mode, self.generateGitAttributes())
1121         else:
1122             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1123
1124 class Command:
1125     def __init__(self):
1126         self.usage = "usage: %prog [options]"
1127         self.needsGit = True
1128         self.verbose = False
1129
1130 class P4UserMap:
1131     def __init__(self):
1132         self.userMapFromPerforceServer = False
1133         self.myP4UserId = None
1134
1135     def p4UserId(self):
1136         if self.myP4UserId:
1137             return self.myP4UserId
1138
1139         results = p4CmdList("user -o")
1140         for r in results:
1141             if r.has_key('User'):
1142                 self.myP4UserId = r['User']
1143                 return r['User']
1144         die("Could not find your p4 user id")
1145
1146     def p4UserIsMe(self, p4User):
1147         # return True if the given p4 user is actually me
1148         me = self.p4UserId()
1149         if not p4User or p4User != me:
1150             return False
1151         else:
1152             return True
1153
1154     def getUserCacheFilename(self):
1155         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1156         return home + "/.gitp4-usercache.txt"
1157
1158     def getUserMapFromPerforceServer(self):
1159         if self.userMapFromPerforceServer:
1160             return
1161         self.users = {}
1162         self.emails = {}
1163
1164         for output in p4CmdList("users"):
1165             if not output.has_key("User"):
1166                 continue
1167             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1168             self.emails[output["Email"]] = output["User"]
1169
1170         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1171         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1172             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1173             if mapUser and len(mapUser[0]) == 3:
1174                 user = mapUser[0][0]
1175                 fullname = mapUser[0][1]
1176                 email = mapUser[0][2]
1177                 self.users[user] = fullname + " <" + email + ">"
1178                 self.emails[email] = user
1179
1180         s = ''
1181         for (key, val) in self.users.items():
1182             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1183
1184         open(self.getUserCacheFilename(), "wb").write(s)
1185         self.userMapFromPerforceServer = True
1186
1187     def loadUserMapFromCache(self):
1188         self.users = {}
1189         self.userMapFromPerforceServer = False
1190         try:
1191             cache = open(self.getUserCacheFilename(), "rb")
1192             lines = cache.readlines()
1193             cache.close()
1194             for line in lines:
1195                 entry = line.strip().split("\t")
1196                 self.users[entry[0]] = entry[1]
1197         except IOError:
1198             self.getUserMapFromPerforceServer()
1199
1200 class P4Debug(Command):
1201     def __init__(self):
1202         Command.__init__(self)
1203         self.options = []
1204         self.description = "A tool to debug the output of p4 -G."
1205         self.needsGit = False
1206
1207     def run(self, args):
1208         j = 0
1209         for output in p4CmdList(args):
1210             print 'Element: %d' % j
1211             j += 1
1212             print output
1213         return True
1214
1215 class P4RollBack(Command):
1216     def __init__(self):
1217         Command.__init__(self)
1218         self.options = [
1219             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1220         ]
1221         self.description = "A tool to debug the multi-branch import. Don't use :)"
1222         self.rollbackLocalBranches = False
1223
1224     def run(self, args):
1225         if len(args) != 1:
1226             return False
1227         maxChange = int(args[0])
1228
1229         if "p4ExitCode" in p4Cmd("changes -m 1"):
1230             die("Problems executing p4");
1231
1232         if self.rollbackLocalBranches:
1233             refPrefix = "refs/heads/"
1234             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1235         else:
1236             refPrefix = "refs/remotes/"
1237             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1238
1239         for line in lines:
1240             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1241                 line = line.strip()
1242                 ref = refPrefix + line
1243                 log = extractLogMessageFromGitCommit(ref)
1244                 settings = extractSettingsGitLog(log)
1245
1246                 depotPaths = settings['depot-paths']
1247                 change = settings['change']
1248
1249                 changed = False
1250
1251                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1252                                                            for p in depotPaths]))) == 0:
1253                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1254                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1255                     continue
1256
1257                 while change and int(change) > maxChange:
1258                     changed = True
1259                     if self.verbose:
1260                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1261                     system("git update-ref %s \"%s^\"" % (ref, ref))
1262                     log = extractLogMessageFromGitCommit(ref)
1263                     settings =  extractSettingsGitLog(log)
1264
1265
1266                     depotPaths = settings['depot-paths']
1267                     change = settings['change']
1268
1269                 if changed:
1270                     print "%s rewound to %s" % (ref, change)
1271
1272         return True
1273
1274 class P4Submit(Command, P4UserMap):
1275
1276     conflict_behavior_choices = ("ask", "skip", "quit")
1277
1278     def __init__(self):
1279         Command.__init__(self)
1280         P4UserMap.__init__(self)
1281         self.options = [
1282                 optparse.make_option("--origin", dest="origin"),
1283                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1284                 # preserve the user, requires relevant p4 permissions
1285                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1286                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1287                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1288                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1289                 optparse.make_option("--conflict", dest="conflict_behavior",
1290                                      choices=self.conflict_behavior_choices),
1291                 optparse.make_option("--branch", dest="branch"),
1292         ]
1293         self.description = "Submit changes from git to the perforce depot."
1294         self.usage += " [name of git branch to submit into perforce depot]"
1295         self.origin = ""
1296         self.detectRenames = False
1297         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1298         self.dry_run = False
1299         self.prepare_p4_only = False
1300         self.conflict_behavior = None
1301         self.isWindows = (platform.system() == "Windows")
1302         self.exportLabels = False
1303         self.p4HasMoveCommand = p4_has_move_command()
1304         self.branch = None
1305
1306         if gitConfig('git-p4.largeFileSystem'):
1307             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1308
1309     def check(self):
1310         if len(p4CmdList("opened ...")) > 0:
1311             die("You have files opened with perforce! Close them before starting the sync.")
1312
1313     def separate_jobs_from_description(self, message):
1314         """Extract and return a possible Jobs field in the commit
1315            message.  It goes into a separate section in the p4 change
1316            specification.
1317
1318            A jobs line starts with "Jobs:" and looks like a new field
1319            in a form.  Values are white-space separated on the same
1320            line or on following lines that start with a tab.
1321
1322            This does not parse and extract the full git commit message
1323            like a p4 form.  It just sees the Jobs: line as a marker
1324            to pass everything from then on directly into the p4 form,
1325            but outside the description section.
1326
1327            Return a tuple (stripped log message, jobs string)."""
1328
1329         m = re.search(r'^Jobs:', message, re.MULTILINE)
1330         if m is None:
1331             return (message, None)
1332
1333         jobtext = message[m.start():]
1334         stripped_message = message[:m.start()].rstrip()
1335         return (stripped_message, jobtext)
1336
1337     def prepareLogMessage(self, template, message, jobs):
1338         """Edits the template returned from "p4 change -o" to insert
1339            the message in the Description field, and the jobs text in
1340            the Jobs field."""
1341         result = ""
1342
1343         inDescriptionSection = False
1344
1345         for line in template.split("\n"):
1346             if line.startswith("#"):
1347                 result += line + "\n"
1348                 continue
1349
1350             if inDescriptionSection:
1351                 if line.startswith("Files:") or line.startswith("Jobs:"):
1352                     inDescriptionSection = False
1353                     # insert Jobs section
1354                     if jobs:
1355                         result += jobs + "\n"
1356                 else:
1357                     continue
1358             else:
1359                 if line.startswith("Description:"):
1360                     inDescriptionSection = True
1361                     line += "\n"
1362                     for messageLine in message.split("\n"):
1363                         line += "\t" + messageLine + "\n"
1364
1365             result += line + "\n"
1366
1367         return result
1368
1369     def patchRCSKeywords(self, file, pattern):
1370         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1371         (handle, outFileName) = tempfile.mkstemp(dir='.')
1372         try:
1373             outFile = os.fdopen(handle, "w+")
1374             inFile = open(file, "r")
1375             regexp = re.compile(pattern, re.VERBOSE)
1376             for line in inFile.readlines():
1377                 line = regexp.sub(r'$\1$', line)
1378                 outFile.write(line)
1379             inFile.close()
1380             outFile.close()
1381             # Forcibly overwrite the original file
1382             os.unlink(file)
1383             shutil.move(outFileName, file)
1384         except:
1385             # cleanup our temporary file
1386             os.unlink(outFileName)
1387             print "Failed to strip RCS keywords in %s" % file
1388             raise
1389
1390         print "Patched up RCS keywords in %s" % file
1391
1392     def p4UserForCommit(self,id):
1393         # Return the tuple (perforce user,git email) for a given git commit id
1394         self.getUserMapFromPerforceServer()
1395         gitEmail = read_pipe(["git", "log", "--max-count=1",
1396                               "--format=%ae", id])
1397         gitEmail = gitEmail.strip()
1398         if not self.emails.has_key(gitEmail):
1399             return (None,gitEmail)
1400         else:
1401             return (self.emails[gitEmail],gitEmail)
1402
1403     def checkValidP4Users(self,commits):
1404         # check if any git authors cannot be mapped to p4 users
1405         for id in commits:
1406             (user,email) = self.p4UserForCommit(id)
1407             if not user:
1408                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1409                 if gitConfigBool("git-p4.allowMissingP4Users"):
1410                     print "%s" % msg
1411                 else:
1412                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1413
1414     def lastP4Changelist(self):
1415         # Get back the last changelist number submitted in this client spec. This
1416         # then gets used to patch up the username in the change. If the same
1417         # client spec is being used by multiple processes then this might go
1418         # wrong.
1419         results = p4CmdList("client -o")        # find the current client
1420         client = None
1421         for r in results:
1422             if r.has_key('Client'):
1423                 client = r['Client']
1424                 break
1425         if not client:
1426             die("could not get client spec")
1427         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1428         for r in results:
1429             if r.has_key('change'):
1430                 return r['change']
1431         die("Could not get changelist number for last submit - cannot patch up user details")
1432
1433     def modifyChangelistUser(self, changelist, newUser):
1434         # fixup the user field of a changelist after it has been submitted.
1435         changes = p4CmdList("change -o %s" % changelist)
1436         if len(changes) != 1:
1437             die("Bad output from p4 change modifying %s to user %s" %
1438                 (changelist, newUser))
1439
1440         c = changes[0]
1441         if c['User'] == newUser: return   # nothing to do
1442         c['User'] = newUser
1443         input = marshal.dumps(c)
1444
1445         result = p4CmdList("change -f -i", stdin=input)
1446         for r in result:
1447             if r.has_key('code'):
1448                 if r['code'] == 'error':
1449                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1450             if r.has_key('data'):
1451                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1452                 return
1453         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1454
1455     def canChangeChangelists(self):
1456         # check to see if we have p4 admin or super-user permissions, either of
1457         # which are required to modify changelists.
1458         results = p4CmdList(["protects", self.depotPath])
1459         for r in results:
1460             if r.has_key('perm'):
1461                 if r['perm'] == 'admin':
1462                     return 1
1463                 if r['perm'] == 'super':
1464                     return 1
1465         return 0
1466
1467     def prepareSubmitTemplate(self):
1468         """Run "p4 change -o" to grab a change specification template.
1469            This does not use "p4 -G", as it is nice to keep the submission
1470            template in original order, since a human might edit it.
1471
1472            Remove lines in the Files section that show changes to files
1473            outside the depot path we're committing into."""
1474
1475         [upstream, settings] = findUpstreamBranchPoint()
1476
1477         template = ""
1478         inFilesSection = False
1479         for line in p4_read_pipe_lines(['change', '-o']):
1480             if line.endswith("\r\n"):
1481                 line = line[:-2] + "\n"
1482             if inFilesSection:
1483                 if line.startswith("\t"):
1484                     # path starts and ends with a tab
1485                     path = line[1:]
1486                     lastTab = path.rfind("\t")
1487                     if lastTab != -1:
1488                         path = path[:lastTab]
1489                         if settings.has_key('depot-paths'):
1490                             if not [p for p in settings['depot-paths']
1491                                     if p4PathStartsWith(path, p)]:
1492                                 continue
1493                         else:
1494                             if not p4PathStartsWith(path, self.depotPath):
1495                                 continue
1496                 else:
1497                     inFilesSection = False
1498             else:
1499                 if line.startswith("Files:"):
1500                     inFilesSection = True
1501
1502             template += line
1503
1504         return template
1505
1506     def edit_template(self, template_file):
1507         """Invoke the editor to let the user change the submission
1508            message.  Return true if okay to continue with the submit."""
1509
1510         # if configured to skip the editing part, just submit
1511         if gitConfigBool("git-p4.skipSubmitEdit"):
1512             return True
1513
1514         # look at the modification time, to check later if the user saved
1515         # the file
1516         mtime = os.stat(template_file).st_mtime
1517
1518         # invoke the editor
1519         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1520             editor = os.environ.get("P4EDITOR")
1521         else:
1522             editor = read_pipe("git var GIT_EDITOR").strip()
1523         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1524
1525         # If the file was not saved, prompt to see if this patch should
1526         # be skipped.  But skip this verification step if configured so.
1527         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1528             return True
1529
1530         # modification time updated means user saved the file
1531         if os.stat(template_file).st_mtime > mtime:
1532             return True
1533
1534         while True:
1535             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1536             if response == 'y':
1537                 return True
1538             if response == 'n':
1539                 return False
1540
1541     def get_diff_description(self, editedFiles, filesToAdd):
1542         # diff
1543         if os.environ.has_key("P4DIFF"):
1544             del(os.environ["P4DIFF"])
1545         diff = ""
1546         for editedFile in editedFiles:
1547             diff += p4_read_pipe(['diff', '-du',
1548                                   wildcard_encode(editedFile)])
1549
1550         # new file diff
1551         newdiff = ""
1552         for newFile in filesToAdd:
1553             newdiff += "==== new file ====\n"
1554             newdiff += "--- /dev/null\n"
1555             newdiff += "+++ %s\n" % newFile
1556             f = open(newFile, "r")
1557             for line in f.readlines():
1558                 newdiff += "+" + line
1559             f.close()
1560
1561         return (diff + newdiff).replace('\r\n', '\n')
1562
1563     def applyCommit(self, id):
1564         """Apply one commit, return True if it succeeded."""
1565
1566         print "Applying", read_pipe(["git", "show", "-s",
1567                                      "--format=format:%h %s", id])
1568
1569         (p4User, gitEmail) = self.p4UserForCommit(id)
1570
1571         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1572         filesToAdd = set()
1573         filesToChangeType = set()
1574         filesToDelete = set()
1575         editedFiles = set()
1576         pureRenameCopy = set()
1577         filesToChangeExecBit = {}
1578
1579         for line in diff:
1580             diff = parseDiffTreeEntry(line)
1581             modifier = diff['status']
1582             path = diff['src']
1583             if modifier == "M":
1584                 p4_edit(path)
1585                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1586                     filesToChangeExecBit[path] = diff['dst_mode']
1587                 editedFiles.add(path)
1588             elif modifier == "A":
1589                 filesToAdd.add(path)
1590                 filesToChangeExecBit[path] = diff['dst_mode']
1591                 if path in filesToDelete:
1592                     filesToDelete.remove(path)
1593             elif modifier == "D":
1594                 filesToDelete.add(path)
1595                 if path in filesToAdd:
1596                     filesToAdd.remove(path)
1597             elif modifier == "C":
1598                 src, dest = diff['src'], diff['dst']
1599                 p4_integrate(src, dest)
1600                 pureRenameCopy.add(dest)
1601                 if diff['src_sha1'] != diff['dst_sha1']:
1602                     p4_edit(dest)
1603                     pureRenameCopy.discard(dest)
1604                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1605                     p4_edit(dest)
1606                     pureRenameCopy.discard(dest)
1607                     filesToChangeExecBit[dest] = diff['dst_mode']
1608                 if self.isWindows:
1609                     # turn off read-only attribute
1610                     os.chmod(dest, stat.S_IWRITE)
1611                 os.unlink(dest)
1612                 editedFiles.add(dest)
1613             elif modifier == "R":
1614                 src, dest = diff['src'], diff['dst']
1615                 if self.p4HasMoveCommand:
1616                     p4_edit(src)        # src must be open before move
1617                     p4_move(src, dest)  # opens for (move/delete, move/add)
1618                 else:
1619                     p4_integrate(src, dest)
1620                     if diff['src_sha1'] != diff['dst_sha1']:
1621                         p4_edit(dest)
1622                     else:
1623                         pureRenameCopy.add(dest)
1624                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1625                     if not self.p4HasMoveCommand:
1626                         p4_edit(dest)   # with move: already open, writable
1627                     filesToChangeExecBit[dest] = diff['dst_mode']
1628                 if not self.p4HasMoveCommand:
1629                     if self.isWindows:
1630                         os.chmod(dest, stat.S_IWRITE)
1631                     os.unlink(dest)
1632                     filesToDelete.add(src)
1633                 editedFiles.add(dest)
1634             elif modifier == "T":
1635                 filesToChangeType.add(path)
1636             else:
1637                 die("unknown modifier %s for %s" % (modifier, path))
1638
1639         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1640         patchcmd = diffcmd + " | git apply "
1641         tryPatchCmd = patchcmd + "--check -"
1642         applyPatchCmd = patchcmd + "--check --apply -"
1643         patch_succeeded = True
1644
1645         if os.system(tryPatchCmd) != 0:
1646             fixed_rcs_keywords = False
1647             patch_succeeded = False
1648             print "Unfortunately applying the change failed!"
1649
1650             # Patch failed, maybe it's just RCS keyword woes. Look through
1651             # the patch to see if that's possible.
1652             if gitConfigBool("git-p4.attemptRCSCleanup"):
1653                 file = None
1654                 pattern = None
1655                 kwfiles = {}
1656                 for file in editedFiles | filesToDelete:
1657                     # did this file's delta contain RCS keywords?
1658                     pattern = p4_keywords_regexp_for_file(file)
1659
1660                     if pattern:
1661                         # this file is a possibility...look for RCS keywords.
1662                         regexp = re.compile(pattern, re.VERBOSE)
1663                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1664                             if regexp.search(line):
1665                                 if verbose:
1666                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1667                                 kwfiles[file] = pattern
1668                                 break
1669
1670                 for file in kwfiles:
1671                     if verbose:
1672                         print "zapping %s with %s" % (line,pattern)
1673                     # File is being deleted, so not open in p4.  Must
1674                     # disable the read-only bit on windows.
1675                     if self.isWindows and file not in editedFiles:
1676                         os.chmod(file, stat.S_IWRITE)
1677                     self.patchRCSKeywords(file, kwfiles[file])
1678                     fixed_rcs_keywords = True
1679
1680             if fixed_rcs_keywords:
1681                 print "Retrying the patch with RCS keywords cleaned up"
1682                 if os.system(tryPatchCmd) == 0:
1683                     patch_succeeded = True
1684
1685         if not patch_succeeded:
1686             for f in editedFiles:
1687                 p4_revert(f)
1688             return False
1689
1690         #
1691         # Apply the patch for real, and do add/delete/+x handling.
1692         #
1693         system(applyPatchCmd)
1694
1695         for f in filesToChangeType:
1696             p4_edit(f, "-t", "auto")
1697         for f in filesToAdd:
1698             p4_add(f)
1699         for f in filesToDelete:
1700             p4_revert(f)
1701             p4_delete(f)
1702
1703         # Set/clear executable bits
1704         for f in filesToChangeExecBit.keys():
1705             mode = filesToChangeExecBit[f]
1706             setP4ExecBit(f, mode)
1707
1708         #
1709         # Build p4 change description, starting with the contents
1710         # of the git commit message.
1711         #
1712         logMessage = extractLogMessageFromGitCommit(id)
1713         logMessage = logMessage.strip()
1714         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1715
1716         template = self.prepareSubmitTemplate()
1717         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1718
1719         if self.preserveUser:
1720            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1721
1722         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1723             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1724             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1725             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1726
1727         separatorLine = "######## everything below this line is just the diff #######\n"
1728         if not self.prepare_p4_only:
1729             submitTemplate += separatorLine
1730             submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1731
1732         (handle, fileName) = tempfile.mkstemp()
1733         tmpFile = os.fdopen(handle, "w+b")
1734         if self.isWindows:
1735             submitTemplate = submitTemplate.replace("\n", "\r\n")
1736         tmpFile.write(submitTemplate)
1737         tmpFile.close()
1738
1739         if self.prepare_p4_only:
1740             #
1741             # Leave the p4 tree prepared, and the submit template around
1742             # and let the user decide what to do next
1743             #
1744             print
1745             print "P4 workspace prepared for submission."
1746             print "To submit or revert, go to client workspace"
1747             print "  " + self.clientPath
1748             print
1749             print "To submit, use \"p4 submit\" to write a new description,"
1750             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1751                   " \"git p4\"." % fileName
1752             print "You can delete the file \"%s\" when finished." % fileName
1753
1754             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1755                 print "To preserve change ownership by user %s, you must\n" \
1756                       "do \"p4 change -f <change>\" after submitting and\n" \
1757                       "edit the User field."
1758             if pureRenameCopy:
1759                 print "After submitting, renamed files must be re-synced."
1760                 print "Invoke \"p4 sync -f\" on each of these files:"
1761                 for f in pureRenameCopy:
1762                     print "  " + f
1763
1764             print
1765             print "To revert the changes, use \"p4 revert ...\", and delete"
1766             print "the submit template file \"%s\"" % fileName
1767             if filesToAdd:
1768                 print "Since the commit adds new files, they must be deleted:"
1769                 for f in filesToAdd:
1770                     print "  " + f
1771             print
1772             return True
1773
1774         #
1775         # Let the user edit the change description, then submit it.
1776         #
1777         submitted = False
1778
1779         try:
1780             if self.edit_template(fileName):
1781                 # read the edited message and submit
1782                 tmpFile = open(fileName, "rb")
1783                 message = tmpFile.read()
1784                 tmpFile.close()
1785                 if self.isWindows:
1786                     message = message.replace("\r\n", "\n")
1787                 submitTemplate = message[:message.index(separatorLine)]
1788                 p4_write_pipe(['submit', '-i'], submitTemplate)
1789
1790                 if self.preserveUser:
1791                     if p4User:
1792                         # Get last changelist number. Cannot easily get it from
1793                         # the submit command output as the output is
1794                         # unmarshalled.
1795                         changelist = self.lastP4Changelist()
1796                         self.modifyChangelistUser(changelist, p4User)
1797
1798                 # The rename/copy happened by applying a patch that created a
1799                 # new file.  This leaves it writable, which confuses p4.
1800                 for f in pureRenameCopy:
1801                     p4_sync(f, "-f")
1802                 submitted = True
1803
1804         finally:
1805             # skip this patch
1806             if not submitted:
1807                 print "Submission cancelled, undoing p4 changes."
1808                 for f in editedFiles:
1809                     p4_revert(f)
1810                 for f in filesToAdd:
1811                     p4_revert(f)
1812                     os.remove(f)
1813                 for f in filesToDelete:
1814                     p4_revert(f)
1815
1816         os.remove(fileName)
1817         return submitted
1818
1819     # Export git tags as p4 labels. Create a p4 label and then tag
1820     # with that.
1821     def exportGitTags(self, gitTags):
1822         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1823         if len(validLabelRegexp) == 0:
1824             validLabelRegexp = defaultLabelRegexp
1825         m = re.compile(validLabelRegexp)
1826
1827         for name in gitTags:
1828
1829             if not m.match(name):
1830                 if verbose:
1831                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1832                 continue
1833
1834             # Get the p4 commit this corresponds to
1835             logMessage = extractLogMessageFromGitCommit(name)
1836             values = extractSettingsGitLog(logMessage)
1837
1838             if not values.has_key('change'):
1839                 # a tag pointing to something not sent to p4; ignore
1840                 if verbose:
1841                     print "git tag %s does not give a p4 commit" % name
1842                 continue
1843             else:
1844                 changelist = values['change']
1845
1846             # Get the tag details.
1847             inHeader = True
1848             isAnnotated = False
1849             body = []
1850             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1851                 l = l.strip()
1852                 if inHeader:
1853                     if re.match(r'tag\s+', l):
1854                         isAnnotated = True
1855                     elif re.match(r'\s*$', l):
1856                         inHeader = False
1857                         continue
1858                 else:
1859                     body.append(l)
1860
1861             if not isAnnotated:
1862                 body = ["lightweight tag imported by git p4\n"]
1863
1864             # Create the label - use the same view as the client spec we are using
1865             clientSpec = getClientSpec()
1866
1867             labelTemplate  = "Label: %s\n" % name
1868             labelTemplate += "Description:\n"
1869             for b in body:
1870                 labelTemplate += "\t" + b + "\n"
1871             labelTemplate += "View:\n"
1872             for depot_side in clientSpec.mappings:
1873                 labelTemplate += "\t%s\n" % depot_side
1874
1875             if self.dry_run:
1876                 print "Would create p4 label %s for tag" % name
1877             elif self.prepare_p4_only:
1878                 print "Not creating p4 label %s for tag due to option" \
1879                       " --prepare-p4-only" % name
1880             else:
1881                 p4_write_pipe(["label", "-i"], labelTemplate)
1882
1883                 # Use the label
1884                 p4_system(["tag", "-l", name] +
1885                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1886
1887                 if verbose:
1888                     print "created p4 label for tag %s" % name
1889
1890     def run(self, args):
1891         if len(args) == 0:
1892             self.master = currentGitBranch()
1893         elif len(args) == 1:
1894             self.master = args[0]
1895             if not branchExists(self.master):
1896                 die("Branch %s does not exist" % self.master)
1897         else:
1898             return False
1899
1900         if self.master:
1901             allowSubmit = gitConfig("git-p4.allowSubmit")
1902             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1903                 die("%s is not in git-p4.allowSubmit" % self.master)
1904
1905         [upstream, settings] = findUpstreamBranchPoint()
1906         self.depotPath = settings['depot-paths'][0]
1907         if len(self.origin) == 0:
1908             self.origin = upstream
1909
1910         if self.preserveUser:
1911             if not self.canChangeChangelists():
1912                 die("Cannot preserve user names without p4 super-user or admin permissions")
1913
1914         # if not set from the command line, try the config file
1915         if self.conflict_behavior is None:
1916             val = gitConfig("git-p4.conflict")
1917             if val:
1918                 if val not in self.conflict_behavior_choices:
1919                     die("Invalid value '%s' for config git-p4.conflict" % val)
1920             else:
1921                 val = "ask"
1922             self.conflict_behavior = val
1923
1924         if self.verbose:
1925             print "Origin branch is " + self.origin
1926
1927         if len(self.depotPath) == 0:
1928             print "Internal error: cannot locate perforce depot path from existing branches"
1929             sys.exit(128)
1930
1931         self.useClientSpec = False
1932         if gitConfigBool("git-p4.useclientspec"):
1933             self.useClientSpec = True
1934         if self.useClientSpec:
1935             self.clientSpecDirs = getClientSpec()
1936
1937         # Check for the existence of P4 branches
1938         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1939
1940         if self.useClientSpec and not branchesDetected:
1941             # all files are relative to the client spec
1942             self.clientPath = getClientRoot()
1943         else:
1944             self.clientPath = p4Where(self.depotPath)
1945
1946         if self.clientPath == "":
1947             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1948
1949         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1950         self.oldWorkingDirectory = os.getcwd()
1951
1952         # ensure the clientPath exists
1953         new_client_dir = False
1954         if not os.path.exists(self.clientPath):
1955             new_client_dir = True
1956             os.makedirs(self.clientPath)
1957
1958         chdir(self.clientPath, is_client_path=True)
1959         if self.dry_run:
1960             print "Would synchronize p4 checkout in %s" % self.clientPath
1961         else:
1962             print "Synchronizing p4 checkout..."
1963             if new_client_dir:
1964                 # old one was destroyed, and maybe nobody told p4
1965                 p4_sync("...", "-f")
1966             else:
1967                 p4_sync("...")
1968         self.check()
1969
1970         commits = []
1971         if self.master:
1972             commitish = self.master
1973         else:
1974             commitish = 'HEAD'
1975
1976         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1977             commits.append(line.strip())
1978         commits.reverse()
1979
1980         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1981             self.checkAuthorship = False
1982         else:
1983             self.checkAuthorship = True
1984
1985         if self.preserveUser:
1986             self.checkValidP4Users(commits)
1987
1988         #
1989         # Build up a set of options to be passed to diff when
1990         # submitting each commit to p4.
1991         #
1992         if self.detectRenames:
1993             # command-line -M arg
1994             self.diffOpts = "-M"
1995         else:
1996             # If not explicitly set check the config variable
1997             detectRenames = gitConfig("git-p4.detectRenames")
1998
1999             if detectRenames.lower() == "false" or detectRenames == "":
2000                 self.diffOpts = ""
2001             elif detectRenames.lower() == "true":
2002                 self.diffOpts = "-M"
2003             else:
2004                 self.diffOpts = "-M%s" % detectRenames
2005
2006         # no command-line arg for -C or --find-copies-harder, just
2007         # config variables
2008         detectCopies = gitConfig("git-p4.detectCopies")
2009         if detectCopies.lower() == "false" or detectCopies == "":
2010             pass
2011         elif detectCopies.lower() == "true":
2012             self.diffOpts += " -C"
2013         else:
2014             self.diffOpts += " -C%s" % detectCopies
2015
2016         if gitConfigBool("git-p4.detectCopiesHarder"):
2017             self.diffOpts += " --find-copies-harder"
2018
2019         #
2020         # Apply the commits, one at a time.  On failure, ask if should
2021         # continue to try the rest of the patches, or quit.
2022         #
2023         if self.dry_run:
2024             print "Would apply"
2025         applied = []
2026         last = len(commits) - 1
2027         for i, commit in enumerate(commits):
2028             if self.dry_run:
2029                 print " ", read_pipe(["git", "show", "-s",
2030                                       "--format=format:%h %s", commit])
2031                 ok = True
2032             else:
2033                 ok = self.applyCommit(commit)
2034             if ok:
2035                 applied.append(commit)
2036             else:
2037                 if self.prepare_p4_only and i < last:
2038                     print "Processing only the first commit due to option" \
2039                           " --prepare-p4-only"
2040                     break
2041                 if i < last:
2042                     quit = False
2043                     while True:
2044                         # prompt for what to do, or use the option/variable
2045                         if self.conflict_behavior == "ask":
2046                             print "What do you want to do?"
2047                             response = raw_input("[s]kip this commit but apply"
2048                                                  " the rest, or [q]uit? ")
2049                             if not response:
2050                                 continue
2051                         elif self.conflict_behavior == "skip":
2052                             response = "s"
2053                         elif self.conflict_behavior == "quit":
2054                             response = "q"
2055                         else:
2056                             die("Unknown conflict_behavior '%s'" %
2057                                 self.conflict_behavior)
2058
2059                         if response[0] == "s":
2060                             print "Skipping this commit, but applying the rest"
2061                             break
2062                         if response[0] == "q":
2063                             print "Quitting"
2064                             quit = True
2065                             break
2066                     if quit:
2067                         break
2068
2069         chdir(self.oldWorkingDirectory)
2070
2071         if self.dry_run:
2072             pass
2073         elif self.prepare_p4_only:
2074             pass
2075         elif len(commits) == len(applied):
2076             print "All commits applied!"
2077
2078             sync = P4Sync()
2079             if self.branch:
2080                 sync.branch = self.branch
2081             sync.run([])
2082
2083             rebase = P4Rebase()
2084             rebase.rebase()
2085
2086         else:
2087             if len(applied) == 0:
2088                 print "No commits applied."
2089             else:
2090                 print "Applied only the commits marked with '*':"
2091                 for c in commits:
2092                     if c in applied:
2093                         star = "*"
2094                     else:
2095                         star = " "
2096                     print star, read_pipe(["git", "show", "-s",
2097                                            "--format=format:%h %s",  c])
2098                 print "You will have to do 'git p4 sync' and rebase."
2099
2100         if gitConfigBool("git-p4.exportLabels"):
2101             self.exportLabels = True
2102
2103         if self.exportLabels:
2104             p4Labels = getP4Labels(self.depotPath)
2105             gitTags = getGitTags()
2106
2107             missingGitTags = gitTags - p4Labels
2108             self.exportGitTags(missingGitTags)
2109
2110         # exit with error unless everything applied perfectly
2111         if len(commits) != len(applied):
2112                 sys.exit(1)
2113
2114         return True
2115
2116 class View(object):
2117     """Represent a p4 view ("p4 help views"), and map files in a
2118        repo according to the view."""
2119
2120     def __init__(self, client_name):
2121         self.mappings = []
2122         self.client_prefix = "//%s/" % client_name
2123         # cache results of "p4 where" to lookup client file locations
2124         self.client_spec_path_cache = {}
2125
2126     def append(self, view_line):
2127         """Parse a view line, splitting it into depot and client
2128            sides.  Append to self.mappings, preserving order.  This
2129            is only needed for tag creation."""
2130
2131         # Split the view line into exactly two words.  P4 enforces
2132         # structure on these lines that simplifies this quite a bit.
2133         #
2134         # Either or both words may be double-quoted.
2135         # Single quotes do not matter.
2136         # Double-quote marks cannot occur inside the words.
2137         # A + or - prefix is also inside the quotes.
2138         # There are no quotes unless they contain a space.
2139         # The line is already white-space stripped.
2140         # The two words are separated by a single space.
2141         #
2142         if view_line[0] == '"':
2143             # First word is double quoted.  Find its end.
2144             close_quote_index = view_line.find('"', 1)
2145             if close_quote_index <= 0:
2146                 die("No first-word closing quote found: %s" % view_line)
2147             depot_side = view_line[1:close_quote_index]
2148             # skip closing quote and space
2149             rhs_index = close_quote_index + 1 + 1
2150         else:
2151             space_index = view_line.find(" ")
2152             if space_index <= 0:
2153                 die("No word-splitting space found: %s" % view_line)
2154             depot_side = view_line[0:space_index]
2155             rhs_index = space_index + 1
2156
2157         # prefix + means overlay on previous mapping
2158         if depot_side.startswith("+"):
2159             depot_side = depot_side[1:]
2160
2161         # prefix - means exclude this path, leave out of mappings
2162         exclude = False
2163         if depot_side.startswith("-"):
2164             exclude = True
2165             depot_side = depot_side[1:]
2166
2167         if not exclude:
2168             self.mappings.append(depot_side)
2169
2170     def convert_client_path(self, clientFile):
2171         # chop off //client/ part to make it relative
2172         if not clientFile.startswith(self.client_prefix):
2173             die("No prefix '%s' on clientFile '%s'" %
2174                 (self.client_prefix, clientFile))
2175         return clientFile[len(self.client_prefix):]
2176
2177     def update_client_spec_path_cache(self, files):
2178         """ Caching file paths by "p4 where" batch query """
2179
2180         # List depot file paths exclude that already cached
2181         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2182
2183         if len(fileArgs) == 0:
2184             return  # All files in cache
2185
2186         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2187         for res in where_result:
2188             if "code" in res and res["code"] == "error":
2189                 # assume error is "... file(s) not in client view"
2190                 continue
2191             if "clientFile" not in res:
2192                 die("No clientFile in 'p4 where' output")
2193             if "unmap" in res:
2194                 # it will list all of them, but only one not unmap-ped
2195                 continue
2196             if gitConfigBool("core.ignorecase"):
2197                 res['depotFile'] = res['depotFile'].lower()
2198             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2199
2200         # not found files or unmap files set to ""
2201         for depotFile in fileArgs:
2202             if gitConfigBool("core.ignorecase"):
2203                 depotFile = depotFile.lower()
2204             if depotFile not in self.client_spec_path_cache:
2205                 self.client_spec_path_cache[depotFile] = ""
2206
2207     def map_in_client(self, depot_path):
2208         """Return the relative location in the client where this
2209            depot file should live.  Returns "" if the file should
2210            not be mapped in the client."""
2211
2212         if gitConfigBool("core.ignorecase"):
2213             depot_path = depot_path.lower()
2214
2215         if depot_path in self.client_spec_path_cache:
2216             return self.client_spec_path_cache[depot_path]
2217
2218         die( "Error: %s is not found in client spec path" % depot_path )
2219         return ""
2220
2221 class P4Sync(Command, P4UserMap):
2222     delete_actions = ( "delete", "move/delete", "purge" )
2223
2224     def __init__(self):
2225         Command.__init__(self)
2226         P4UserMap.__init__(self)
2227         self.options = [
2228                 optparse.make_option("--branch", dest="branch"),
2229                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2230                 optparse.make_option("--changesfile", dest="changesFile"),
2231                 optparse.make_option("--silent", dest="silent", action="store_true"),
2232                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2233                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2234                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2235                                      help="Import into refs/heads/ , not refs/remotes"),
2236                 optparse.make_option("--max-changes", dest="maxChanges",
2237                                      help="Maximum number of changes to import"),
2238                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2239                                      help="Internal block size to use when iteratively calling p4 changes"),
2240                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2241                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2242                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2243                                      help="Only sync files that are included in the Perforce Client Spec"),
2244                 optparse.make_option("-/", dest="cloneExclude",
2245                                      action="append", type="string",
2246                                      help="exclude depot path"),
2247         ]
2248         self.description = """Imports from Perforce into a git repository.\n
2249     example:
2250     //depot/my/project/ -- to import the current head
2251     //depot/my/project/@all -- to import everything
2252     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2253
2254     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2255
2256         self.usage += " //depot/path[@revRange]"
2257         self.silent = False
2258         self.createdBranches = set()
2259         self.committedChanges = set()
2260         self.branch = ""
2261         self.detectBranches = False
2262         self.detectLabels = False
2263         self.importLabels = False
2264         self.changesFile = ""
2265         self.syncWithOrigin = True
2266         self.importIntoRemotes = True
2267         self.maxChanges = ""
2268         self.changes_block_size = None
2269         self.keepRepoPath = False
2270         self.depotPaths = None
2271         self.p4BranchesInGit = []
2272         self.cloneExclude = []
2273         self.useClientSpec = False
2274         self.useClientSpec_from_options = False
2275         self.clientSpecDirs = None
2276         self.tempBranches = []
2277         self.tempBranchLocation = "refs/git-p4-tmp"
2278         self.largeFileSystem = None
2279
2280         if gitConfig('git-p4.largeFileSystem'):
2281             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2282             self.largeFileSystem = largeFileSystemConstructor(
2283                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2284             )
2285
2286         if gitConfig("git-p4.syncFromOrigin") == "false":
2287             self.syncWithOrigin = False
2288
2289     # This is required for the "append" cloneExclude action
2290     def ensure_value(self, attr, value):
2291         if not hasattr(self, attr) or getattr(self, attr) is None:
2292             setattr(self, attr, value)
2293         return getattr(self, attr)
2294
2295     # Force a checkpoint in fast-import and wait for it to finish
2296     def checkpoint(self):
2297         self.gitStream.write("checkpoint\n\n")
2298         self.gitStream.write("progress checkpoint\n\n")
2299         out = self.gitOutput.readline()
2300         if self.verbose:
2301             print "checkpoint finished: " + out
2302
2303     def extractFilesFromCommit(self, commit):
2304         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2305                              for path in self.cloneExclude]
2306         files = []
2307         fnum = 0
2308         while commit.has_key("depotFile%s" % fnum):
2309             path =  commit["depotFile%s" % fnum]
2310
2311             if [p for p in self.cloneExclude
2312                 if p4PathStartsWith(path, p)]:
2313                 found = False
2314             else:
2315                 found = [p for p in self.depotPaths
2316                          if p4PathStartsWith(path, p)]
2317             if not found:
2318                 fnum = fnum + 1
2319                 continue
2320
2321             file = {}
2322             file["path"] = path
2323             file["rev"] = commit["rev%s" % fnum]
2324             file["action"] = commit["action%s" % fnum]
2325             file["type"] = commit["type%s" % fnum]
2326             files.append(file)
2327             fnum = fnum + 1
2328         return files
2329
2330     def extractJobsFromCommit(self, commit):
2331         jobs = []
2332         jnum = 0
2333         while commit.has_key("job%s" % jnum):
2334             job = commit["job%s" % jnum]
2335             jobs.append(job)
2336             jnum = jnum + 1
2337         return jobs
2338
2339     def stripRepoPath(self, path, prefixes):
2340         """When streaming files, this is called to map a p4 depot path
2341            to where it should go in git.  The prefixes are either
2342            self.depotPaths, or self.branchPrefixes in the case of
2343            branch detection."""
2344
2345         if self.useClientSpec:
2346             # branch detection moves files up a level (the branch name)
2347             # from what client spec interpretation gives
2348             path = self.clientSpecDirs.map_in_client(path)
2349             if self.detectBranches:
2350                 for b in self.knownBranches:
2351                     if path.startswith(b + "/"):
2352                         path = path[len(b)+1:]
2353
2354         elif self.keepRepoPath:
2355             # Preserve everything in relative path name except leading
2356             # //depot/; just look at first prefix as they all should
2357             # be in the same depot.
2358             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2359             if p4PathStartsWith(path, depot):
2360                 path = path[len(depot):]
2361
2362         else:
2363             for p in prefixes:
2364                 if p4PathStartsWith(path, p):
2365                     path = path[len(p):]
2366                     break
2367
2368         path = wildcard_decode(path)
2369         return path
2370
2371     def splitFilesIntoBranches(self, commit):
2372         """Look at each depotFile in the commit to figure out to what
2373            branch it belongs."""
2374
2375         if self.clientSpecDirs:
2376             files = self.extractFilesFromCommit(commit)
2377             self.clientSpecDirs.update_client_spec_path_cache(files)
2378
2379         branches = {}
2380         fnum = 0
2381         while commit.has_key("depotFile%s" % fnum):
2382             path =  commit["depotFile%s" % fnum]
2383             found = [p for p in self.depotPaths
2384                      if p4PathStartsWith(path, p)]
2385             if not found:
2386                 fnum = fnum + 1
2387                 continue
2388
2389             file = {}
2390             file["path"] = path
2391             file["rev"] = commit["rev%s" % fnum]
2392             file["action"] = commit["action%s" % fnum]
2393             file["type"] = commit["type%s" % fnum]
2394             fnum = fnum + 1
2395
2396             # start with the full relative path where this file would
2397             # go in a p4 client
2398             if self.useClientSpec:
2399                 relPath = self.clientSpecDirs.map_in_client(path)
2400             else:
2401                 relPath = self.stripRepoPath(path, self.depotPaths)
2402
2403             for branch in self.knownBranches.keys():
2404                 # add a trailing slash so that a commit into qt/4.2foo
2405                 # doesn't end up in qt/4.2, e.g.
2406                 if relPath.startswith(branch + "/"):
2407                     if branch not in branches:
2408                         branches[branch] = []
2409                     branches[branch].append(file)
2410                     break
2411
2412         return branches
2413
2414     def writeToGitStream(self, gitMode, relPath, contents):
2415         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2416         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2417         for d in contents:
2418             self.gitStream.write(d)
2419         self.gitStream.write('\n')
2420
2421     # output one file from the P4 stream
2422     # - helper for streamP4Files
2423
2424     def streamOneP4File(self, file, contents):
2425         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2426         if verbose:
2427             size = int(self.stream_file['fileSize'])
2428             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2429             sys.stdout.flush()
2430
2431         (type_base, type_mods) = split_p4_type(file["type"])
2432
2433         git_mode = "100644"
2434         if "x" in type_mods:
2435             git_mode = "100755"
2436         if type_base == "symlink":
2437             git_mode = "120000"
2438             # p4 print on a symlink sometimes contains "target\n";
2439             # if it does, remove the newline
2440             data = ''.join(contents)
2441             if not data:
2442                 # Some version of p4 allowed creating a symlink that pointed
2443                 # to nothing.  This causes p4 errors when checking out such
2444                 # a change, and errors here too.  Work around it by ignoring
2445                 # the bad symlink; hopefully a future change fixes it.
2446                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2447                 return
2448             elif data[-1] == '\n':
2449                 contents = [data[:-1]]
2450             else:
2451                 contents = [data]
2452
2453         if type_base == "utf16":
2454             # p4 delivers different text in the python output to -G
2455             # than it does when using "print -o", or normal p4 client
2456             # operations.  utf16 is converted to ascii or utf8, perhaps.
2457             # But ascii text saved as -t utf16 is completely mangled.
2458             # Invoke print -o to get the real contents.
2459             #
2460             # On windows, the newlines will always be mangled by print, so put
2461             # them back too.  This is not needed to the cygwin windows version,
2462             # just the native "NT" type.
2463             #
2464             try:
2465                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2466             except Exception as e:
2467                 if 'Translation of file content failed' in str(e):
2468                     type_base = 'binary'
2469                 else:
2470                     raise e
2471             else:
2472                 if p4_version_string().find('/NT') >= 0:
2473                     text = text.replace('\r\n', '\n')
2474                 contents = [ text ]
2475
2476         if type_base == "apple":
2477             # Apple filetype files will be streamed as a concatenation of
2478             # its appledouble header and the contents.  This is useless
2479             # on both macs and non-macs.  If using "print -q -o xx", it
2480             # will create "xx" with the data, and "%xx" with the header.
2481             # This is also not very useful.
2482             #
2483             # Ideally, someday, this script can learn how to generate
2484             # appledouble files directly and import those to git, but
2485             # non-mac machines can never find a use for apple filetype.
2486             print "\nIgnoring apple filetype file %s" % file['depotFile']
2487             return
2488
2489         # Note that we do not try to de-mangle keywords on utf16 files,
2490         # even though in theory somebody may want that.
2491         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2492         if pattern:
2493             regexp = re.compile(pattern, re.VERBOSE)
2494             text = ''.join(contents)
2495             text = regexp.sub(r'$\1$', text)
2496             contents = [ text ]
2497
2498         try:
2499             relPath.decode('ascii')
2500         except:
2501             encoding = 'utf8'
2502             if gitConfig('git-p4.pathEncoding'):
2503                 encoding = gitConfig('git-p4.pathEncoding')
2504             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2505             if self.verbose:
2506                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2507
2508         if self.largeFileSystem:
2509             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2510
2511         self.writeToGitStream(git_mode, relPath, contents)
2512
2513     def streamOneP4Deletion(self, file):
2514         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2515         if verbose:
2516             sys.stdout.write("delete %s\n" % relPath)
2517             sys.stdout.flush()
2518         self.gitStream.write("D %s\n" % relPath)
2519
2520         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2521             self.largeFileSystem.removeLargeFile(relPath)
2522
2523     # handle another chunk of streaming data
2524     def streamP4FilesCb(self, marshalled):
2525
2526         # catch p4 errors and complain
2527         err = None
2528         if "code" in marshalled:
2529             if marshalled["code"] == "error":
2530                 if "data" in marshalled:
2531                     err = marshalled["data"].rstrip()
2532
2533         if not err and 'fileSize' in self.stream_file:
2534             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2535             if required_bytes > 0:
2536                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2537                     os.getcwd(), required_bytes/1024/1024
2538                 )
2539
2540         if err:
2541             f = None
2542             if self.stream_have_file_info:
2543                 if "depotFile" in self.stream_file:
2544                     f = self.stream_file["depotFile"]
2545             # force a failure in fast-import, else an empty
2546             # commit will be made
2547             self.gitStream.write("\n")
2548             self.gitStream.write("die-now\n")
2549             self.gitStream.close()
2550             # ignore errors, but make sure it exits first
2551             self.importProcess.wait()
2552             if f:
2553                 die("Error from p4 print for %s: %s" % (f, err))
2554             else:
2555                 die("Error from p4 print: %s" % err)
2556
2557         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2558             # start of a new file - output the old one first
2559             self.streamOneP4File(self.stream_file, self.stream_contents)
2560             self.stream_file = {}
2561             self.stream_contents = []
2562             self.stream_have_file_info = False
2563
2564         # pick up the new file information... for the
2565         # 'data' field we need to append to our array
2566         for k in marshalled.keys():
2567             if k == 'data':
2568                 if 'streamContentSize' not in self.stream_file:
2569                     self.stream_file['streamContentSize'] = 0
2570                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2571                 self.stream_contents.append(marshalled['data'])
2572             else:
2573                 self.stream_file[k] = marshalled[k]
2574
2575         if (verbose and
2576             'streamContentSize' in self.stream_file and
2577             'fileSize' in self.stream_file and
2578             'depotFile' in self.stream_file):
2579             size = int(self.stream_file["fileSize"])
2580             if size > 0:
2581                 progress = 100*self.stream_file['streamContentSize']/size
2582                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2583                 sys.stdout.flush()
2584
2585         self.stream_have_file_info = True
2586
2587     # Stream directly from "p4 files" into "git fast-import"
2588     def streamP4Files(self, files):
2589         filesForCommit = []
2590         filesToRead = []
2591         filesToDelete = []
2592
2593         for f in files:
2594             filesForCommit.append(f)
2595             if f['action'] in self.delete_actions:
2596                 filesToDelete.append(f)
2597             else:
2598                 filesToRead.append(f)
2599
2600         # deleted files...
2601         for f in filesToDelete:
2602             self.streamOneP4Deletion(f)
2603
2604         if len(filesToRead) > 0:
2605             self.stream_file = {}
2606             self.stream_contents = []
2607             self.stream_have_file_info = False
2608
2609             # curry self argument
2610             def streamP4FilesCbSelf(entry):
2611                 self.streamP4FilesCb(entry)
2612
2613             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2614
2615             p4CmdList(["-x", "-", "print"],
2616                       stdin=fileArgs,
2617                       cb=streamP4FilesCbSelf)
2618
2619             # do the last chunk
2620             if self.stream_file.has_key('depotFile'):
2621                 self.streamOneP4File(self.stream_file, self.stream_contents)
2622
2623     def make_email(self, userid):
2624         if userid in self.users:
2625             return self.users[userid]
2626         else:
2627             return "%s <a@b>" % userid
2628
2629     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2630         """ Stream a p4 tag.
2631         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2632         """
2633
2634         if verbose:
2635             print "writing tag %s for commit %s" % (labelName, commit)
2636         gitStream.write("tag %s\n" % labelName)
2637         gitStream.write("from %s\n" % commit)
2638
2639         if labelDetails.has_key('Owner'):
2640             owner = labelDetails["Owner"]
2641         else:
2642             owner = None
2643
2644         # Try to use the owner of the p4 label, or failing that,
2645         # the current p4 user id.
2646         if owner:
2647             email = self.make_email(owner)
2648         else:
2649             email = self.make_email(self.p4UserId())
2650         tagger = "%s %s %s" % (email, epoch, self.tz)
2651
2652         gitStream.write("tagger %s\n" % tagger)
2653
2654         print "labelDetails=",labelDetails
2655         if labelDetails.has_key('Description'):
2656             description = labelDetails['Description']
2657         else:
2658             description = 'Label from git p4'
2659
2660         gitStream.write("data %d\n" % len(description))
2661         gitStream.write(description)
2662         gitStream.write("\n")
2663
2664     def inClientSpec(self, path):
2665         if not self.clientSpecDirs:
2666             return True
2667         inClientSpec = self.clientSpecDirs.map_in_client(path)
2668         if not inClientSpec and self.verbose:
2669             print('Ignoring file outside of client spec: {0}'.format(path))
2670         return inClientSpec
2671
2672     def hasBranchPrefix(self, path):
2673         if not self.branchPrefixes:
2674             return True
2675         hasPrefix = [p for p in self.branchPrefixes
2676                         if p4PathStartsWith(path, p)]
2677         if not hasPrefix and self.verbose:
2678             print('Ignoring file outside of prefix: {0}'.format(path))
2679         return hasPrefix
2680
2681     def commit(self, details, files, branch, parent = ""):
2682         epoch = details["time"]
2683         author = details["user"]
2684         jobs = self.extractJobsFromCommit(details)
2685
2686         if self.verbose:
2687             print('commit into {0}'.format(branch))
2688
2689         if self.clientSpecDirs:
2690             self.clientSpecDirs.update_client_spec_path_cache(files)
2691
2692         files = [f for f in files
2693             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2694
2695         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2696             print('Ignoring revision {0} as it would produce an empty commit.'
2697                 .format(details['change']))
2698             return
2699
2700         self.gitStream.write("commit %s\n" % branch)
2701         self.gitStream.write("mark :%s\n" % details["change"])
2702         self.committedChanges.add(int(details["change"]))
2703         committer = ""
2704         if author not in self.users:
2705             self.getUserMapFromPerforceServer()
2706         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2707
2708         self.gitStream.write("committer %s\n" % committer)
2709
2710         self.gitStream.write("data <<EOT\n")
2711         self.gitStream.write(details["desc"])
2712         if len(jobs) > 0:
2713             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2714         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2715                              (','.join(self.branchPrefixes), details["change"]))
2716         if len(details['options']) > 0:
2717             self.gitStream.write(": options = %s" % details['options'])
2718         self.gitStream.write("]\nEOT\n\n")
2719
2720         if len(parent) > 0:
2721             if self.verbose:
2722                 print "parent %s" % parent
2723             self.gitStream.write("from %s\n" % parent)
2724
2725         self.streamP4Files(files)
2726         self.gitStream.write("\n")
2727
2728         change = int(details["change"])
2729
2730         if self.labels.has_key(change):
2731             label = self.labels[change]
2732             labelDetails = label[0]
2733             labelRevisions = label[1]
2734             if self.verbose:
2735                 print "Change %s is labelled %s" % (change, labelDetails)
2736
2737             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2738                                                 for p in self.branchPrefixes])
2739
2740             if len(files) == len(labelRevisions):
2741
2742                 cleanedFiles = {}
2743                 for info in files:
2744                     if info["action"] in self.delete_actions:
2745                         continue
2746                     cleanedFiles[info["depotFile"]] = info["rev"]
2747
2748                 if cleanedFiles == labelRevisions:
2749                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2750
2751                 else:
2752                     if not self.silent:
2753                         print ("Tag %s does not match with change %s: files do not match."
2754                                % (labelDetails["label"], change))
2755
2756             else:
2757                 if not self.silent:
2758                     print ("Tag %s does not match with change %s: file count is different."
2759                            % (labelDetails["label"], change))
2760
2761     # Build a dictionary of changelists and labels, for "detect-labels" option.
2762     def getLabels(self):
2763         self.labels = {}
2764
2765         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2766         if len(l) > 0 and not self.silent:
2767             print "Finding files belonging to labels in %s" % `self.depotPaths`
2768
2769         for output in l:
2770             label = output["label"]
2771             revisions = {}
2772             newestChange = 0
2773             if self.verbose:
2774                 print "Querying files for label %s" % label
2775             for file in p4CmdList(["files"] +
2776                                       ["%s...@%s" % (p, label)
2777                                           for p in self.depotPaths]):
2778                 revisions[file["depotFile"]] = file["rev"]
2779                 change = int(file["change"])
2780                 if change > newestChange:
2781                     newestChange = change
2782
2783             self.labels[newestChange] = [output, revisions]
2784
2785         if self.verbose:
2786             print "Label changes: %s" % self.labels.keys()
2787
2788     # Import p4 labels as git tags. A direct mapping does not
2789     # exist, so assume that if all the files are at the same revision
2790     # then we can use that, or it's something more complicated we should
2791     # just ignore.
2792     def importP4Labels(self, stream, p4Labels):
2793         if verbose:
2794             print "import p4 labels: " + ' '.join(p4Labels)
2795
2796         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2797         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2798         if len(validLabelRegexp) == 0:
2799             validLabelRegexp = defaultLabelRegexp
2800         m = re.compile(validLabelRegexp)
2801
2802         for name in p4Labels:
2803             commitFound = False
2804
2805             if not m.match(name):
2806                 if verbose:
2807                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2808                 continue
2809
2810             if name in ignoredP4Labels:
2811                 continue
2812
2813             labelDetails = p4CmdList(['label', "-o", name])[0]
2814
2815             # get the most recent changelist for each file in this label
2816             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2817                                 for p in self.depotPaths])
2818
2819             if change.has_key('change'):
2820                 # find the corresponding git commit; take the oldest commit
2821                 changelist = int(change['change'])
2822                 if changelist in self.committedChanges:
2823                     gitCommit = ":%d" % changelist       # use a fast-import mark
2824                     commitFound = True
2825                 else:
2826                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2827                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2828                     if len(gitCommit) == 0:
2829                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2830                     else:
2831                         commitFound = True
2832                         gitCommit = gitCommit.strip()
2833
2834                 if commitFound:
2835                     # Convert from p4 time format
2836                     try:
2837                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2838                     except ValueError:
2839                         print "Could not convert label time %s" % labelDetails['Update']
2840                         tmwhen = 1
2841
2842                     when = int(time.mktime(tmwhen))
2843                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2844                     if verbose:
2845                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2846             else:
2847                 if verbose:
2848                     print "Label %s has no changelists - possibly deleted?" % name
2849
2850             if not commitFound:
2851                 # We can't import this label; don't try again as it will get very
2852                 # expensive repeatedly fetching all the files for labels that will
2853                 # never be imported. If the label is moved in the future, the
2854                 # ignore will need to be removed manually.
2855                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2856
2857     def guessProjectName(self):
2858         for p in self.depotPaths:
2859             if p.endswith("/"):
2860                 p = p[:-1]
2861             p = p[p.strip().rfind("/") + 1:]
2862             if not p.endswith("/"):
2863                p += "/"
2864             return p
2865
2866     def getBranchMapping(self):
2867         lostAndFoundBranches = set()
2868
2869         user = gitConfig("git-p4.branchUser")
2870         if len(user) > 0:
2871             command = "branches -u %s" % user
2872         else:
2873             command = "branches"
2874
2875         for info in p4CmdList(command):
2876             details = p4Cmd(["branch", "-o", info["branch"]])
2877             viewIdx = 0
2878             while details.has_key("View%s" % viewIdx):
2879                 paths = details["View%s" % viewIdx].split(" ")
2880                 viewIdx = viewIdx + 1
2881                 # require standard //depot/foo/... //depot/bar/... mapping
2882                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2883                     continue
2884                 source = paths[0]
2885                 destination = paths[1]
2886                 ## HACK
2887                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2888                     source = source[len(self.depotPaths[0]):-4]
2889                     destination = destination[len(self.depotPaths[0]):-4]
2890
2891                     if destination in self.knownBranches:
2892                         if not self.silent:
2893                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2894                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2895                         continue
2896
2897                     self.knownBranches[destination] = source
2898
2899                     lostAndFoundBranches.discard(destination)
2900
2901                     if source not in self.knownBranches:
2902                         lostAndFoundBranches.add(source)
2903
2904         # Perforce does not strictly require branches to be defined, so we also
2905         # check git config for a branch list.
2906         #
2907         # Example of branch definition in git config file:
2908         # [git-p4]
2909         #   branchList=main:branchA
2910         #   branchList=main:branchB
2911         #   branchList=branchA:branchC
2912         configBranches = gitConfigList("git-p4.branchList")
2913         for branch in configBranches:
2914             if branch:
2915                 (source, destination) = branch.split(":")
2916                 self.knownBranches[destination] = source
2917
2918                 lostAndFoundBranches.discard(destination)
2919
2920                 if source not in self.knownBranches:
2921                     lostAndFoundBranches.add(source)
2922
2923
2924         for branch in lostAndFoundBranches:
2925             self.knownBranches[branch] = branch
2926
2927     def getBranchMappingFromGitBranches(self):
2928         branches = p4BranchesInGit(self.importIntoRemotes)
2929         for branch in branches.keys():
2930             if branch == "master":
2931                 branch = "main"
2932             else:
2933                 branch = branch[len(self.projectName):]
2934             self.knownBranches[branch] = branch
2935
2936     def updateOptionDict(self, d):
2937         option_keys = {}
2938         if self.keepRepoPath:
2939             option_keys['keepRepoPath'] = 1
2940
2941         d["options"] = ' '.join(sorted(option_keys.keys()))
2942
2943     def readOptions(self, d):
2944         self.keepRepoPath = (d.has_key('options')
2945                              and ('keepRepoPath' in d['options']))
2946
2947     def gitRefForBranch(self, branch):
2948         if branch == "main":
2949             return self.refPrefix + "master"
2950
2951         if len(branch) <= 0:
2952             return branch
2953
2954         return self.refPrefix + self.projectName + branch
2955
2956     def gitCommitByP4Change(self, ref, change):
2957         if self.verbose:
2958             print "looking in ref " + ref + " for change %s using bisect..." % change
2959
2960         earliestCommit = ""
2961         latestCommit = parseRevision(ref)
2962
2963         while True:
2964             if self.verbose:
2965                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2966             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2967             if len(next) == 0:
2968                 if self.verbose:
2969                     print "argh"
2970                 return ""
2971             log = extractLogMessageFromGitCommit(next)
2972             settings = extractSettingsGitLog(log)
2973             currentChange = int(settings['change'])
2974             if self.verbose:
2975                 print "current change %s" % currentChange
2976
2977             if currentChange == change:
2978                 if self.verbose:
2979                     print "found %s" % next
2980                 return next
2981
2982             if currentChange < change:
2983                 earliestCommit = "^%s" % next
2984             else:
2985                 latestCommit = "%s" % next
2986
2987         return ""
2988
2989     def importNewBranch(self, branch, maxChange):
2990         # make fast-import flush all changes to disk and update the refs using the checkpoint
2991         # command so that we can try to find the branch parent in the git history
2992         self.gitStream.write("checkpoint\n\n");
2993         self.gitStream.flush();
2994         branchPrefix = self.depotPaths[0] + branch + "/"
2995         range = "@1,%s" % maxChange
2996         #print "prefix" + branchPrefix
2997         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2998         if len(changes) <= 0:
2999             return False
3000         firstChange = changes[0]
3001         #print "first change in branch: %s" % firstChange
3002         sourceBranch = self.knownBranches[branch]
3003         sourceDepotPath = self.depotPaths[0] + sourceBranch
3004         sourceRef = self.gitRefForBranch(sourceBranch)
3005         #print "source " + sourceBranch
3006
3007         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3008         #print "branch parent: %s" % branchParentChange
3009         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3010         if len(gitParent) > 0:
3011             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3012             #print "parent git commit: %s" % gitParent
3013
3014         self.importChanges(changes)
3015         return True
3016
3017     def searchParent(self, parent, branch, target):
3018         parentFound = False
3019         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3020                                      "--no-merges", parent]):
3021             blob = blob.strip()
3022             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3023                 parentFound = True
3024                 if self.verbose:
3025                     print "Found parent of %s in commit %s" % (branch, blob)
3026                 break
3027         if parentFound:
3028             return blob
3029         else:
3030             return None
3031
3032     def importChanges(self, changes):
3033         cnt = 1
3034         for change in changes:
3035             description = p4_describe(change)
3036             self.updateOptionDict(description)
3037
3038             if not self.silent:
3039                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3040                 sys.stdout.flush()
3041             cnt = cnt + 1
3042
3043             try:
3044                 if self.detectBranches:
3045                     branches = self.splitFilesIntoBranches(description)
3046                     for branch in branches.keys():
3047                         ## HACK  --hwn
3048                         branchPrefix = self.depotPaths[0] + branch + "/"
3049                         self.branchPrefixes = [ branchPrefix ]
3050
3051                         parent = ""
3052
3053                         filesForCommit = branches[branch]
3054
3055                         if self.verbose:
3056                             print "branch is %s" % branch
3057
3058                         self.updatedBranches.add(branch)
3059
3060                         if branch not in self.createdBranches:
3061                             self.createdBranches.add(branch)
3062                             parent = self.knownBranches[branch]
3063                             if parent == branch:
3064                                 parent = ""
3065                             else:
3066                                 fullBranch = self.projectName + branch
3067                                 if fullBranch not in self.p4BranchesInGit:
3068                                     if not self.silent:
3069                                         print("\n    Importing new branch %s" % fullBranch);
3070                                     if self.importNewBranch(branch, change - 1):
3071                                         parent = ""
3072                                         self.p4BranchesInGit.append(fullBranch)
3073                                     if not self.silent:
3074                                         print("\n    Resuming with change %s" % change);
3075
3076                                 if self.verbose:
3077                                     print "parent determined through known branches: %s" % parent
3078
3079                         branch = self.gitRefForBranch(branch)
3080                         parent = self.gitRefForBranch(parent)
3081
3082                         if self.verbose:
3083                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3084
3085                         if len(parent) == 0 and branch in self.initialParents:
3086                             parent = self.initialParents[branch]
3087                             del self.initialParents[branch]
3088
3089                         blob = None
3090                         if len(parent) > 0:
3091                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3092                             if self.verbose:
3093                                 print "Creating temporary branch: " + tempBranch
3094                             self.commit(description, filesForCommit, tempBranch)
3095                             self.tempBranches.append(tempBranch)
3096                             self.checkpoint()
3097                             blob = self.searchParent(parent, branch, tempBranch)
3098                         if blob:
3099                             self.commit(description, filesForCommit, branch, blob)
3100                         else:
3101                             if self.verbose:
3102                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3103                             self.commit(description, filesForCommit, branch, parent)
3104                 else:
3105                     files = self.extractFilesFromCommit(description)
3106                     self.commit(description, files, self.branch,
3107                                 self.initialParent)
3108                     # only needed once, to connect to the previous commit
3109                     self.initialParent = ""
3110             except IOError:
3111                 print self.gitError.read()
3112                 sys.exit(1)
3113
3114     def importHeadRevision(self, revision):
3115         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3116
3117         details = {}
3118         details["user"] = "git perforce import user"
3119         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3120                            % (' '.join(self.depotPaths), revision))
3121         details["change"] = revision
3122         newestRevision = 0
3123
3124         fileCnt = 0
3125         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3126
3127         for info in p4CmdList(["files"] + fileArgs):
3128
3129             if 'code' in info and info['code'] == 'error':
3130                 sys.stderr.write("p4 returned an error: %s\n"
3131                                  % info['data'])
3132                 if info['data'].find("must refer to client") >= 0:
3133                     sys.stderr.write("This particular p4 error is misleading.\n")
3134                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3135                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3136                 sys.exit(1)
3137             if 'p4ExitCode' in info:
3138                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3139                 sys.exit(1)
3140
3141
3142             change = int(info["change"])
3143             if change > newestRevision:
3144                 newestRevision = change
3145
3146             if info["action"] in self.delete_actions:
3147                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3148                 #fileCnt = fileCnt + 1
3149                 continue
3150
3151             for prop in ["depotFile", "rev", "action", "type" ]:
3152                 details["%s%s" % (prop, fileCnt)] = info[prop]
3153
3154             fileCnt = fileCnt + 1
3155
3156         details["change"] = newestRevision
3157
3158         # Use time from top-most change so that all git p4 clones of
3159         # the same p4 repo have the same commit SHA1s.
3160         res = p4_describe(newestRevision)
3161         details["time"] = res["time"]
3162
3163         self.updateOptionDict(details)
3164         try:
3165             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3166         except IOError:
3167             print "IO error with git fast-import. Is your git version recent enough?"
3168             print self.gitError.read()
3169
3170
3171     def run(self, args):
3172         self.depotPaths = []
3173         self.changeRange = ""
3174         self.previousDepotPaths = []
3175         self.hasOrigin = False
3176
3177         # map from branch depot path to parent branch
3178         self.knownBranches = {}
3179         self.initialParents = {}
3180
3181         if self.importIntoRemotes:
3182             self.refPrefix = "refs/remotes/p4/"
3183         else:
3184             self.refPrefix = "refs/heads/p4/"
3185
3186         if self.syncWithOrigin:
3187             self.hasOrigin = originP4BranchesExist()
3188             if self.hasOrigin:
3189                 if not self.silent:
3190                     print 'Syncing with origin first, using "git fetch origin"'
3191                 system("git fetch origin")
3192
3193         branch_arg_given = bool(self.branch)
3194         if len(self.branch) == 0:
3195             self.branch = self.refPrefix + "master"
3196             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3197                 system("git update-ref %s refs/heads/p4" % self.branch)
3198                 system("git branch -D p4")
3199
3200         # accept either the command-line option, or the configuration variable
3201         if self.useClientSpec:
3202             # will use this after clone to set the variable
3203             self.useClientSpec_from_options = True
3204         else:
3205             if gitConfigBool("git-p4.useclientspec"):
3206                 self.useClientSpec = True
3207         if self.useClientSpec:
3208             self.clientSpecDirs = getClientSpec()
3209
3210         # TODO: should always look at previous commits,
3211         # merge with previous imports, if possible.
3212         if args == []:
3213             if self.hasOrigin:
3214                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3215
3216             # branches holds mapping from branch name to sha1
3217             branches = p4BranchesInGit(self.importIntoRemotes)
3218
3219             # restrict to just this one, disabling detect-branches
3220             if branch_arg_given:
3221                 short = self.branch.split("/")[-1]
3222                 if short in branches:
3223                     self.p4BranchesInGit = [ short ]
3224             else:
3225                 self.p4BranchesInGit = branches.keys()
3226
3227             if len(self.p4BranchesInGit) > 1:
3228                 if not self.silent:
3229                     print "Importing from/into multiple branches"
3230                 self.detectBranches = True
3231                 for branch in branches.keys():
3232                     self.initialParents[self.refPrefix + branch] = \
3233                         branches[branch]
3234
3235             if self.verbose:
3236                 print "branches: %s" % self.p4BranchesInGit
3237
3238             p4Change = 0
3239             for branch in self.p4BranchesInGit:
3240                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3241
3242                 settings = extractSettingsGitLog(logMsg)
3243
3244                 self.readOptions(settings)
3245                 if (settings.has_key('depot-paths')
3246                     and settings.has_key ('change')):
3247                     change = int(settings['change']) + 1
3248                     p4Change = max(p4Change, change)
3249
3250                     depotPaths = sorted(settings['depot-paths'])
3251                     if self.previousDepotPaths == []:
3252                         self.previousDepotPaths = depotPaths
3253                     else:
3254                         paths = []
3255                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3256                             prev_list = prev.split("/")
3257                             cur_list = cur.split("/")
3258                             for i in range(0, min(len(cur_list), len(prev_list))):
3259                                 if cur_list[i] <> prev_list[i]:
3260                                     i = i - 1
3261                                     break
3262
3263                             paths.append ("/".join(cur_list[:i + 1]))
3264
3265                         self.previousDepotPaths = paths
3266
3267             if p4Change > 0:
3268                 self.depotPaths = sorted(self.previousDepotPaths)
3269                 self.changeRange = "@%s,#head" % p4Change
3270                 if not self.silent and not self.detectBranches:
3271                     print "Performing incremental import into %s git branch" % self.branch
3272
3273         # accept multiple ref name abbreviations:
3274         #    refs/foo/bar/branch -> use it exactly
3275         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3276         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3277         if not self.branch.startswith("refs/"):
3278             if self.importIntoRemotes:
3279                 prepend = "refs/remotes/"
3280             else:
3281                 prepend = "refs/heads/"
3282             if not self.branch.startswith("p4/"):
3283                 prepend += "p4/"
3284             self.branch = prepend + self.branch
3285
3286         if len(args) == 0 and self.depotPaths:
3287             if not self.silent:
3288                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3289         else:
3290             if self.depotPaths and self.depotPaths != args:
3291                 print ("previous import used depot path %s and now %s was specified. "
3292                        "This doesn't work!" % (' '.join (self.depotPaths),
3293                                                ' '.join (args)))
3294                 sys.exit(1)
3295
3296             self.depotPaths = sorted(args)
3297
3298         revision = ""
3299         self.users = {}
3300
3301         # Make sure no revision specifiers are used when --changesfile
3302         # is specified.
3303         bad_changesfile = False
3304         if len(self.changesFile) > 0:
3305             for p in self.depotPaths:
3306                 if p.find("@") >= 0 or p.find("#") >= 0:
3307                     bad_changesfile = True
3308                     break
3309         if bad_changesfile:
3310             die("Option --changesfile is incompatible with revision specifiers")
3311
3312         newPaths = []
3313         for p in self.depotPaths:
3314             if p.find("@") != -1:
3315                 atIdx = p.index("@")
3316                 self.changeRange = p[atIdx:]
3317                 if self.changeRange == "@all":
3318                     self.changeRange = ""
3319                 elif ',' not in self.changeRange:
3320                     revision = self.changeRange
3321                     self.changeRange = ""
3322                 p = p[:atIdx]
3323             elif p.find("#") != -1:
3324                 hashIdx = p.index("#")
3325                 revision = p[hashIdx:]
3326                 p = p[:hashIdx]
3327             elif self.previousDepotPaths == []:
3328                 # pay attention to changesfile, if given, else import
3329                 # the entire p4 tree at the head revision
3330                 if len(self.changesFile) == 0:
3331                     revision = "#head"
3332
3333             p = re.sub ("\.\.\.$", "", p)
3334             if not p.endswith("/"):
3335                 p += "/"
3336
3337             newPaths.append(p)
3338
3339         self.depotPaths = newPaths
3340
3341         # --detect-branches may change this for each branch
3342         self.branchPrefixes = self.depotPaths
3343
3344         self.loadUserMapFromCache()
3345         self.labels = {}
3346         if self.detectLabels:
3347             self.getLabels();
3348
3349         if self.detectBranches:
3350             ## FIXME - what's a P4 projectName ?
3351             self.projectName = self.guessProjectName()
3352
3353             if self.hasOrigin:
3354                 self.getBranchMappingFromGitBranches()
3355             else:
3356                 self.getBranchMapping()
3357             if self.verbose:
3358                 print "p4-git branches: %s" % self.p4BranchesInGit
3359                 print "initial parents: %s" % self.initialParents
3360             for b in self.p4BranchesInGit:
3361                 if b != "master":
3362
3363                     ## FIXME
3364                     b = b[len(self.projectName):]
3365                 self.createdBranches.add(b)
3366
3367         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3368
3369         self.importProcess = subprocess.Popen(["git", "fast-import"],
3370                                               stdin=subprocess.PIPE,
3371                                               stdout=subprocess.PIPE,
3372                                               stderr=subprocess.PIPE);
3373         self.gitOutput = self.importProcess.stdout
3374         self.gitStream = self.importProcess.stdin
3375         self.gitError = self.importProcess.stderr
3376
3377         if revision:
3378             self.importHeadRevision(revision)
3379         else:
3380             changes = []
3381
3382             if len(self.changesFile) > 0:
3383                 output = open(self.changesFile).readlines()
3384                 changeSet = set()
3385                 for line in output:
3386                     changeSet.add(int(line))
3387
3388                 for change in changeSet:
3389                     changes.append(change)
3390
3391                 changes.sort()
3392             else:
3393                 # catch "git p4 sync" with no new branches, in a repo that
3394                 # does not have any existing p4 branches
3395                 if len(args) == 0:
3396                     if not self.p4BranchesInGit:
3397                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3398
3399                     # The default branch is master, unless --branch is used to
3400                     # specify something else.  Make sure it exists, or complain
3401                     # nicely about how to use --branch.
3402                     if not self.detectBranches:
3403                         if not branch_exists(self.branch):
3404                             if branch_arg_given:
3405                                 die("Error: branch %s does not exist." % self.branch)
3406                             else:
3407                                 die("Error: no branch %s; perhaps specify one with --branch." %
3408                                     self.branch)
3409
3410                 if self.verbose:
3411                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3412                                                               self.changeRange)
3413                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3414
3415                 if len(self.maxChanges) > 0:
3416                     changes = changes[:min(int(self.maxChanges), len(changes))]
3417
3418             if len(changes) == 0:
3419                 if not self.silent:
3420                     print "No changes to import!"
3421             else:
3422                 if not self.silent and not self.detectBranches:
3423                     print "Import destination: %s" % self.branch
3424
3425                 self.updatedBranches = set()
3426
3427                 if not self.detectBranches:
3428                     if args:
3429                         # start a new branch
3430                         self.initialParent = ""
3431                     else:
3432                         # build on a previous revision
3433                         self.initialParent = parseRevision(self.branch)
3434
3435                 self.importChanges(changes)
3436
3437                 if not self.silent:
3438                     print ""
3439                     if len(self.updatedBranches) > 0:
3440                         sys.stdout.write("Updated branches: ")
3441                         for b in self.updatedBranches:
3442                             sys.stdout.write("%s " % b)
3443                         sys.stdout.write("\n")
3444
3445         if gitConfigBool("git-p4.importLabels"):
3446             self.importLabels = True
3447
3448         if self.importLabels:
3449             p4Labels = getP4Labels(self.depotPaths)
3450             gitTags = getGitTags()
3451
3452             missingP4Labels = p4Labels - gitTags
3453             self.importP4Labels(self.gitStream, missingP4Labels)
3454
3455         self.gitStream.close()
3456         if self.importProcess.wait() != 0:
3457             die("fast-import failed: %s" % self.gitError.read())
3458         self.gitOutput.close()
3459         self.gitError.close()
3460
3461         # Cleanup temporary branches created during import
3462         if self.tempBranches != []:
3463             for branch in self.tempBranches:
3464                 read_pipe("git update-ref -d %s" % branch)
3465             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3466
3467         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3468         # a convenient shortcut refname "p4".
3469         if self.importIntoRemotes:
3470             head_ref = self.refPrefix + "HEAD"
3471             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3472                 system(["git", "symbolic-ref", head_ref, self.branch])
3473
3474         return True
3475
3476 class P4Rebase(Command):
3477     def __init__(self):
3478         Command.__init__(self)
3479         self.options = [
3480                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3481         ]
3482         self.importLabels = False
3483         self.description = ("Fetches the latest revision from perforce and "
3484                             + "rebases the current work (branch) against it")
3485
3486     def run(self, args):
3487         sync = P4Sync()
3488         sync.importLabels = self.importLabels
3489         sync.run([])
3490
3491         return self.rebase()
3492
3493     def rebase(self):
3494         if os.system("git update-index --refresh") != 0:
3495             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3496         if len(read_pipe("git diff-index HEAD --")) > 0:
3497             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3498
3499         [upstream, settings] = findUpstreamBranchPoint()
3500         if len(upstream) == 0:
3501             die("Cannot find upstream branchpoint for rebase")
3502
3503         # the branchpoint may be p4/foo~3, so strip off the parent
3504         upstream = re.sub("~[0-9]+$", "", upstream)
3505
3506         print "Rebasing the current branch onto %s" % upstream
3507         oldHead = read_pipe("git rev-parse HEAD").strip()
3508         system("git rebase %s" % upstream)
3509         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3510         return True
3511
3512 class P4Clone(P4Sync):
3513     def __init__(self):
3514         P4Sync.__init__(self)
3515         self.description = "Creates a new git repository and imports from Perforce into it"
3516         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3517         self.options += [
3518             optparse.make_option("--destination", dest="cloneDestination",
3519                                  action='store', default=None,
3520                                  help="where to leave result of the clone"),
3521             optparse.make_option("--bare", dest="cloneBare",
3522                                  action="store_true", default=False),
3523         ]
3524         self.cloneDestination = None
3525         self.needsGit = False
3526         self.cloneBare = False
3527
3528     def defaultDestination(self, args):
3529         ## TODO: use common prefix of args?
3530         depotPath = args[0]
3531         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3532         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3533         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3534         depotDir = re.sub(r"/$", "", depotDir)
3535         return os.path.split(depotDir)[1]
3536
3537     def run(self, args):
3538         if len(args) < 1:
3539             return False
3540
3541         if self.keepRepoPath and not self.cloneDestination:
3542             sys.stderr.write("Must specify destination for --keep-path\n")
3543             sys.exit(1)
3544
3545         depotPaths = args
3546
3547         if not self.cloneDestination and len(depotPaths) > 1:
3548             self.cloneDestination = depotPaths[-1]
3549             depotPaths = depotPaths[:-1]
3550
3551         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3552         for p in depotPaths:
3553             if not p.startswith("//"):
3554                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3555                 return False
3556
3557         if not self.cloneDestination:
3558             self.cloneDestination = self.defaultDestination(args)
3559
3560         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3561
3562         if not os.path.exists(self.cloneDestination):
3563             os.makedirs(self.cloneDestination)
3564         chdir(self.cloneDestination)
3565
3566         init_cmd = [ "git", "init" ]
3567         if self.cloneBare:
3568             init_cmd.append("--bare")
3569         retcode = subprocess.call(init_cmd)
3570         if retcode:
3571             raise CalledProcessError(retcode, init_cmd)
3572
3573         if not P4Sync.run(self, depotPaths):
3574             return False
3575
3576         # create a master branch and check out a work tree
3577         if gitBranchExists(self.branch):
3578             system([ "git", "branch", "master", self.branch ])
3579             if not self.cloneBare:
3580                 system([ "git", "checkout", "-f" ])
3581         else:
3582             print 'Not checking out any branch, use ' \
3583                   '"git checkout -q -b master <branch>"'
3584
3585         # auto-set this variable if invoked with --use-client-spec
3586         if self.useClientSpec_from_options:
3587             system("git config --bool git-p4.useclientspec true")
3588
3589         return True
3590
3591 class P4Branches(Command):
3592     def __init__(self):
3593         Command.__init__(self)
3594         self.options = [ ]
3595         self.description = ("Shows the git branches that hold imports and their "
3596                             + "corresponding perforce depot paths")
3597         self.verbose = False
3598
3599     def run(self, args):
3600         if originP4BranchesExist():
3601             createOrUpdateBranchesFromOrigin()
3602
3603         cmdline = "git rev-parse --symbolic "
3604         cmdline += " --remotes"
3605
3606         for line in read_pipe_lines(cmdline):
3607             line = line.strip()
3608
3609             if not line.startswith('p4/') or line == "p4/HEAD":
3610                 continue
3611             branch = line
3612
3613             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3614             settings = extractSettingsGitLog(log)
3615
3616             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3617         return True
3618
3619 class HelpFormatter(optparse.IndentedHelpFormatter):
3620     def __init__(self):
3621         optparse.IndentedHelpFormatter.__init__(self)
3622
3623     def format_description(self, description):
3624         if description:
3625             return description + "\n"
3626         else:
3627             return ""
3628
3629 def printUsage(commands):
3630     print "usage: %s <command> [options]" % sys.argv[0]
3631     print ""
3632     print "valid commands: %s" % ", ".join(commands)
3633     print ""
3634     print "Try %s <command> --help for command specific help." % sys.argv[0]
3635     print ""
3636
3637 commands = {
3638     "debug" : P4Debug,
3639     "submit" : P4Submit,
3640     "commit" : P4Submit,
3641     "sync" : P4Sync,
3642     "rebase" : P4Rebase,
3643     "clone" : P4Clone,
3644     "rollback" : P4RollBack,
3645     "branches" : P4Branches
3646 }
3647
3648
3649 def main():
3650     if len(sys.argv[1:]) == 0:
3651         printUsage(commands.keys())
3652         sys.exit(2)
3653
3654     cmdName = sys.argv[1]
3655     try:
3656         klass = commands[cmdName]
3657         cmd = klass()
3658     except KeyError:
3659         print "unknown command %s" % cmdName
3660         print ""
3661         printUsage(commands.keys())
3662         sys.exit(2)
3663
3664     options = cmd.options
3665     cmd.gitdir = os.environ.get("GIT_DIR", None)
3666
3667     args = sys.argv[2:]
3668
3669     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3670     if cmd.needsGit:
3671         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3672
3673     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3674                                    options,
3675                                    description = cmd.description,
3676                                    formatter = HelpFormatter())
3677
3678     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3679     global verbose
3680     verbose = cmd.verbose
3681     if cmd.needsGit:
3682         if cmd.gitdir == None:
3683             cmd.gitdir = os.path.abspath(".git")
3684             if not isValidGitDir(cmd.gitdir):
3685                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3686                 if os.path.exists(cmd.gitdir):
3687                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3688                     if len(cdup) > 0:
3689                         chdir(cdup);
3690
3691         if not isValidGitDir(cmd.gitdir):
3692             if isValidGitDir(cmd.gitdir + "/.git"):
3693                 cmd.gitdir += "/.git"
3694             else:
3695                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3696
3697         os.environ["GIT_DIR"] = cmd.gitdir
3698
3699     if not cmd.run(args):
3700         parser.print_help()
3701         sys.exit(2)
3702
3703
3704 if __name__ == '__main__':
3705     main()