Merge branch 'js/mingw-msdn-url'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 # support basestring in python3
31 try:
32     unicode = unicode
33 except NameError:
34     # 'unicode' is undefined, must be Python 3
35     str = str
36     unicode = str
37     bytes = bytes
38     basestring = (str,bytes)
39 else:
40     # 'unicode' exists, must be Python 2
41     str = str
42     unicode = unicode
43     bytes = str
44     basestring = basestring
45
46 try:
47     from subprocess import CalledProcessError
48 except ImportError:
49     # from python2.7:subprocess.py
50     # Exception classes used by this module.
51     class CalledProcessError(Exception):
52         """This exception is raised when a process run by check_call() returns
53         a non-zero exit status.  The exit status will be stored in the
54         returncode attribute."""
55         def __init__(self, returncode, cmd):
56             self.returncode = returncode
57             self.cmd = cmd
58         def __str__(self):
59             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
61 verbose = False
62
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
65
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
68
69 p4_access_checked = False
70
71 def p4_build_cmd(cmd):
72     """Build a suitable p4 command line.
73
74     This consolidates building and returning a p4 command line into one
75     location. It means that hooking into the environment, or other configuration
76     can be done more easily.
77     """
78     real_cmd = ["p4"]
79
80     user = gitConfig("git-p4.user")
81     if len(user) > 0:
82         real_cmd += ["-u",user]
83
84     password = gitConfig("git-p4.password")
85     if len(password) > 0:
86         real_cmd += ["-P", password]
87
88     port = gitConfig("git-p4.port")
89     if len(port) > 0:
90         real_cmd += ["-p", port]
91
92     host = gitConfig("git-p4.host")
93     if len(host) > 0:
94         real_cmd += ["-H", host]
95
96     client = gitConfig("git-p4.client")
97     if len(client) > 0:
98         real_cmd += ["-c", client]
99
100     retries = gitConfigInt("git-p4.retries")
101     if retries is None:
102         # Perform 3 retries by default
103         retries = 3
104     if retries > 0:
105         # Provide a way to not pass this option by setting git-p4.retries to 0
106         real_cmd += ["-r", str(retries)]
107
108     if isinstance(cmd,basestring):
109         real_cmd = ' '.join(real_cmd) + ' ' + cmd
110     else:
111         real_cmd += cmd
112
113     # now check that we can actually talk to the server
114     global p4_access_checked
115     if not p4_access_checked:
116         p4_access_checked = True    # suppress access checks in p4_check_access itself
117         p4_check_access()
118
119     return real_cmd
120
121 def git_dir(path):
122     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123         This won't automatically add ".git" to a directory.
124     """
125     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126     if not d or len(d) == 0:
127         return None
128     else:
129         return d
130
131 def chdir(path, is_client_path=False):
132     """Do chdir to the given path, and set the PWD environment
133        variable for use by P4.  It does not look at getcwd() output.
134        Since we're not using the shell, it is necessary to set the
135        PWD environment variable explicitly.
136
137        Normally, expand the path to force it to be absolute.  This
138        addresses the use of relative path names inside P4 settings,
139        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
140        as given; it looks for .p4config using PWD.
141
142        If is_client_path, the path was handed to us directly by p4,
143        and may be a symbolic link.  Do not call os.getcwd() in this
144        case, because it will cause p4 to think that PWD is not inside
145        the client path.
146        """
147
148     os.chdir(path)
149     if not is_client_path:
150         path = os.getcwd()
151     os.environ['PWD'] = path
152
153 def calcDiskFree():
154     """Return free space in bytes on the disk of the given dirname."""
155     if platform.system() == 'Windows':
156         free_bytes = ctypes.c_ulonglong(0)
157         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158         return free_bytes.value
159     else:
160         st = os.statvfs(os.getcwd())
161         return st.f_bavail * st.f_frsize
162
163 def die(msg):
164     if verbose:
165         raise Exception(msg)
166     else:
167         sys.stderr.write(msg + "\n")
168         sys.exit(1)
169
170 def write_pipe(c, stdin):
171     if verbose:
172         sys.stderr.write('Writing pipe: %s\n' % str(c))
173
174     expand = isinstance(c,basestring)
175     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176     pipe = p.stdin
177     val = pipe.write(stdin)
178     pipe.close()
179     if p.wait():
180         die('Command failed: %s' % str(c))
181
182     return val
183
184 def p4_write_pipe(c, stdin):
185     real_cmd = p4_build_cmd(c)
186     return write_pipe(real_cmd, stdin)
187
188 def read_pipe_full(c):
189     """ Read output from  command. Returns a tuple
190         of the return status, stdout text and stderr
191         text.
192     """
193     if verbose:
194         sys.stderr.write('Reading pipe: %s\n' % str(c))
195
196     expand = isinstance(c,basestring)
197     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198     (out, err) = p.communicate()
199     return (p.returncode, out, err)
200
201 def read_pipe(c, ignore_error=False):
202     """ Read output from  command. Returns the output text on
203         success. On failure, terminates execution, unless
204         ignore_error is True, when it returns an empty string.
205     """
206     (retcode, out, err) = read_pipe_full(c)
207     if retcode != 0:
208         if ignore_error:
209             out = ""
210         else:
211             die('Command failed: %s\nError: %s' % (str(c), err))
212     return out
213
214 def read_pipe_text(c):
215     """ Read output from a command with trailing whitespace stripped.
216         On error, returns None.
217     """
218     (retcode, out, err) = read_pipe_full(c)
219     if retcode != 0:
220         return None
221     else:
222         return out.rstrip()
223
224 def p4_read_pipe(c, ignore_error=False):
225     real_cmd = p4_build_cmd(c)
226     return read_pipe(real_cmd, ignore_error)
227
228 def read_pipe_lines(c):
229     if verbose:
230         sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232     expand = isinstance(c, basestring)
233     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234     pipe = p.stdout
235     val = pipe.readlines()
236     if pipe.close() or p.wait():
237         die('Command failed: %s' % str(c))
238
239     return val
240
241 def p4_read_pipe_lines(c):
242     """Specifically invoke p4 on the command supplied. """
243     real_cmd = p4_build_cmd(c)
244     return read_pipe_lines(real_cmd)
245
246 def p4_has_command(cmd):
247     """Ask p4 for help on this command.  If it returns an error, the
248        command does not exist in this version of p4."""
249     real_cmd = p4_build_cmd(["help", cmd])
250     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251                                    stderr=subprocess.PIPE)
252     p.communicate()
253     return p.returncode == 0
254
255 def p4_has_move_command():
256     """See if the move command exists, that it supports -k, and that
257        it has not been administratively disabled.  The arguments
258        must be correct, but the filenames do not have to exist.  Use
259        ones with wildcards so even if they exist, it will fail."""
260
261     if not p4_has_command("move"):
262         return False
263     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265     (out, err) = p.communicate()
266     # return code will be 1 in either case
267     if err.find("Invalid option") >= 0:
268         return False
269     if err.find("disabled") >= 0:
270         return False
271     # assume it failed because @... was invalid changelist
272     return True
273
274 def system(cmd, ignore_error=False):
275     expand = isinstance(cmd,basestring)
276     if verbose:
277         sys.stderr.write("executing %s\n" % str(cmd))
278     retcode = subprocess.call(cmd, shell=expand)
279     if retcode and not ignore_error:
280         raise CalledProcessError(retcode, cmd)
281
282     return retcode
283
284 def p4_system(cmd):
285     """Specifically invoke p4 as the system command. """
286     real_cmd = p4_build_cmd(cmd)
287     expand = isinstance(real_cmd, basestring)
288     retcode = subprocess.call(real_cmd, shell=expand)
289     if retcode:
290         raise CalledProcessError(retcode, real_cmd)
291
292 def die_bad_access(s):
293     die("failure accessing depot: {0}".format(s.rstrip()))
294
295 def p4_check_access(min_expiration=1):
296     """ Check if we can access Perforce - account still logged in
297     """
298     results = p4CmdList(["login", "-s"])
299
300     if len(results) == 0:
301         # should never get here: always get either some results, or a p4ExitCode
302         assert("could not parse response from perforce")
303
304     result = results[0]
305
306     if 'p4ExitCode' in result:
307         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308         die_bad_access("could not run p4")
309
310     code = result.get("code")
311     if not code:
312         # we get here if we couldn't connect and there was nothing to unmarshal
313         die_bad_access("could not connect")
314
315     elif code == "stat":
316         expiry = result.get("TicketExpiration")
317         if expiry:
318             expiry = int(expiry)
319             if expiry > min_expiration:
320                 # ok to carry on
321                 return
322             else:
323                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325         else:
326             # account without a timeout - all ok
327             return
328
329     elif code == "error":
330         data = result.get("data")
331         if data:
332             die_bad_access("p4 error: {0}".format(data))
333         else:
334             die_bad_access("unknown error")
335     else:
336         die_bad_access("unknown error code {0}".format(code))
337
338 _p4_version_string = None
339 def p4_version_string():
340     """Read the version string, showing just the last line, which
341        hopefully is the interesting version bit.
342
343        $ p4 -V
344        Perforce - The Fast Software Configuration Management System.
345        Copyright 1995-2011 Perforce Software.  All rights reserved.
346        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
347     """
348     global _p4_version_string
349     if not _p4_version_string:
350         a = p4_read_pipe_lines(["-V"])
351         _p4_version_string = a[-1].rstrip()
352     return _p4_version_string
353
354 def p4_integrate(src, dest):
355     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
356
357 def p4_sync(f, *options):
358     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
359
360 def p4_add(f):
361     # forcibly add file names with wildcards
362     if wildcard_present(f):
363         p4_system(["add", "-f", f])
364     else:
365         p4_system(["add", f])
366
367 def p4_delete(f):
368     p4_system(["delete", wildcard_encode(f)])
369
370 def p4_edit(f, *options):
371     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
372
373 def p4_revert(f):
374     p4_system(["revert", wildcard_encode(f)])
375
376 def p4_reopen(type, f):
377     p4_system(["reopen", "-t", type, wildcard_encode(f)])
378
379 def p4_reopen_in_change(changelist, files):
380     cmd = ["reopen", "-c", str(changelist)] + files
381     p4_system(cmd)
382
383 def p4_move(src, dest):
384     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
385
386 def p4_last_change():
387     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
388     return int(results[0]['change'])
389
390 def p4_describe(change, shelved=False):
391     """Make sure it returns a valid result by checking for
392        the presence of field "time".  Return a dict of the
393        results."""
394
395     cmd = ["describe", "-s"]
396     if shelved:
397         cmd += ["-S"]
398     cmd += [str(change)]
399
400     ds = p4CmdList(cmd, skip_info=True)
401     if len(ds) != 1:
402         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
403
404     d = ds[0]
405
406     if "p4ExitCode" in d:
407         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
408                                                       str(d)))
409     if "code" in d:
410         if d["code"] == "error":
411             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
412
413     if "time" not in d:
414         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
415
416     return d
417
418 #
419 # Canonicalize the p4 type and return a tuple of the
420 # base type, plus any modifiers.  See "p4 help filetypes"
421 # for a list and explanation.
422 #
423 def split_p4_type(p4type):
424
425     p4_filetypes_historical = {
426         "ctempobj": "binary+Sw",
427         "ctext": "text+C",
428         "cxtext": "text+Cx",
429         "ktext": "text+k",
430         "kxtext": "text+kx",
431         "ltext": "text+F",
432         "tempobj": "binary+FSw",
433         "ubinary": "binary+F",
434         "uresource": "resource+F",
435         "uxbinary": "binary+Fx",
436         "xbinary": "binary+x",
437         "xltext": "text+Fx",
438         "xtempobj": "binary+Swx",
439         "xtext": "text+x",
440         "xunicode": "unicode+x",
441         "xutf16": "utf16+x",
442     }
443     if p4type in p4_filetypes_historical:
444         p4type = p4_filetypes_historical[p4type]
445     mods = ""
446     s = p4type.split("+")
447     base = s[0]
448     mods = ""
449     if len(s) > 1:
450         mods = s[1]
451     return (base, mods)
452
453 #
454 # return the raw p4 type of a file (text, text+ko, etc)
455 #
456 def p4_type(f):
457     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
458     return results[0]['headType']
459
460 #
461 # Given a type base and modifier, return a regexp matching
462 # the keywords that can be expanded in the file
463 #
464 def p4_keywords_regexp_for_type(base, type_mods):
465     if base in ("text", "unicode", "binary"):
466         kwords = None
467         if "ko" in type_mods:
468             kwords = 'Id|Header'
469         elif "k" in type_mods:
470             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
471         else:
472             return None
473         pattern = r"""
474             \$              # Starts with a dollar, followed by...
475             (%s)            # one of the keywords, followed by...
476             (:[^$\n]+)?     # possibly an old expansion, followed by...
477             \$              # another dollar
478             """ % kwords
479         return pattern
480     else:
481         return None
482
483 #
484 # Given a file, return a regexp matching the possible
485 # RCS keywords that will be expanded, or None for files
486 # with kw expansion turned off.
487 #
488 def p4_keywords_regexp_for_file(file):
489     if not os.path.exists(file):
490         return None
491     else:
492         (type_base, type_mods) = split_p4_type(p4_type(file))
493         return p4_keywords_regexp_for_type(type_base, type_mods)
494
495 def setP4ExecBit(file, mode):
496     # Reopens an already open file and changes the execute bit to match
497     # the execute bit setting in the passed in mode.
498
499     p4Type = "+x"
500
501     if not isModeExec(mode):
502         p4Type = getP4OpenedType(file)
503         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
504         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
505         if p4Type[-1] == "+":
506             p4Type = p4Type[0:-1]
507
508     p4_reopen(p4Type, file)
509
510 def getP4OpenedType(file):
511     # Returns the perforce file type for the given file.
512
513     result = p4_read_pipe(["opened", wildcard_encode(file)])
514     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
515     if match:
516         return match.group(1)
517     else:
518         die("Could not determine file type for %s (result: '%s')" % (file, result))
519
520 # Return the set of all p4 labels
521 def getP4Labels(depotPaths):
522     labels = set()
523     if isinstance(depotPaths,basestring):
524         depotPaths = [depotPaths]
525
526     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
527         label = l['label']
528         labels.add(label)
529
530     return labels
531
532 # Return the set of all git tags
533 def getGitTags():
534     gitTags = set()
535     for line in read_pipe_lines(["git", "tag"]):
536         tag = line.strip()
537         gitTags.add(tag)
538     return gitTags
539
540 def diffTreePattern():
541     # This is a simple generator for the diff tree regex pattern. This could be
542     # a class variable if this and parseDiffTreeEntry were a part of a class.
543     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
544     while True:
545         yield pattern
546
547 def parseDiffTreeEntry(entry):
548     """Parses a single diff tree entry into its component elements.
549
550     See git-diff-tree(1) manpage for details about the format of the diff
551     output. This method returns a dictionary with the following elements:
552
553     src_mode - The mode of the source file
554     dst_mode - The mode of the destination file
555     src_sha1 - The sha1 for the source file
556     dst_sha1 - The sha1 fr the destination file
557     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
558     status_score - The score for the status (applicable for 'C' and 'R'
559                    statuses). This is None if there is no score.
560     src - The path for the source file.
561     dst - The path for the destination file. This is only present for
562           copy or renames. If it is not present, this is None.
563
564     If the pattern is not matched, None is returned."""
565
566     match = diffTreePattern().next().match(entry)
567     if match:
568         return {
569             'src_mode': match.group(1),
570             'dst_mode': match.group(2),
571             'src_sha1': match.group(3),
572             'dst_sha1': match.group(4),
573             'status': match.group(5),
574             'status_score': match.group(6),
575             'src': match.group(7),
576             'dst': match.group(10)
577         }
578     return None
579
580 def isModeExec(mode):
581     # Returns True if the given git mode represents an executable file,
582     # otherwise False.
583     return mode[-3:] == "755"
584
585 class P4Exception(Exception):
586     """ Base class for exceptions from the p4 client """
587     def __init__(self, exit_code):
588         self.p4ExitCode = exit_code
589
590 class P4ServerException(P4Exception):
591     """ Base class for exceptions where we get some kind of marshalled up result from the server """
592     def __init__(self, exit_code, p4_result):
593         super(P4ServerException, self).__init__(exit_code)
594         self.p4_result = p4_result
595         self.code = p4_result[0]['code']
596         self.data = p4_result[0]['data']
597
598 class P4RequestSizeException(P4ServerException):
599     """ One of the maxresults or maxscanrows errors """
600     def __init__(self, exit_code, p4_result, limit):
601         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
602         self.limit = limit
603
604 def isModeExecChanged(src_mode, dst_mode):
605     return isModeExec(src_mode) != isModeExec(dst_mode)
606
607 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
608         errors_as_exceptions=False):
609
610     if isinstance(cmd,basestring):
611         cmd = "-G " + cmd
612         expand = True
613     else:
614         cmd = ["-G"] + cmd
615         expand = False
616
617     cmd = p4_build_cmd(cmd)
618     if verbose:
619         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
620
621     # Use a temporary file to avoid deadlocks without
622     # subprocess.communicate(), which would put another copy
623     # of stdout into memory.
624     stdin_file = None
625     if stdin is not None:
626         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
627         if isinstance(stdin,basestring):
628             stdin_file.write(stdin)
629         else:
630             for i in stdin:
631                 stdin_file.write(i + '\n')
632         stdin_file.flush()
633         stdin_file.seek(0)
634
635     p4 = subprocess.Popen(cmd,
636                           shell=expand,
637                           stdin=stdin_file,
638                           stdout=subprocess.PIPE)
639
640     result = []
641     try:
642         while True:
643             entry = marshal.load(p4.stdout)
644             if skip_info:
645                 if 'code' in entry and entry['code'] == 'info':
646                     continue
647             if cb is not None:
648                 cb(entry)
649             else:
650                 result.append(entry)
651     except EOFError:
652         pass
653     exitCode = p4.wait()
654     if exitCode != 0:
655         if errors_as_exceptions:
656             if len(result) > 0:
657                 data = result[0].get('data')
658                 if data:
659                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
660                     if not m:
661                         m = re.search('Request too large \(over (\d+)\)', data)
662
663                     if m:
664                         limit = int(m.group(1))
665                         raise P4RequestSizeException(exitCode, result, limit)
666
667                 raise P4ServerException(exitCode, result)
668             else:
669                 raise P4Exception(exitCode)
670         else:
671             entry = {}
672             entry["p4ExitCode"] = exitCode
673             result.append(entry)
674
675     return result
676
677 def p4Cmd(cmd):
678     list = p4CmdList(cmd)
679     result = {}
680     for entry in list:
681         result.update(entry)
682     return result;
683
684 def p4Where(depotPath):
685     if not depotPath.endswith("/"):
686         depotPath += "/"
687     depotPathLong = depotPath + "..."
688     outputList = p4CmdList(["where", depotPathLong])
689     output = None
690     for entry in outputList:
691         if "depotFile" in entry:
692             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
693             # The base path always ends with "/...".
694             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
695                 output = entry
696                 break
697         elif "data" in entry:
698             data = entry.get("data")
699             space = data.find(" ")
700             if data[:space] == depotPath:
701                 output = entry
702                 break
703     if output == None:
704         return ""
705     if output["code"] == "error":
706         return ""
707     clientPath = ""
708     if "path" in output:
709         clientPath = output.get("path")
710     elif "data" in output:
711         data = output.get("data")
712         lastSpace = data.rfind(" ")
713         clientPath = data[lastSpace + 1:]
714
715     if clientPath.endswith("..."):
716         clientPath = clientPath[:-3]
717     return clientPath
718
719 def currentGitBranch():
720     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
721
722 def isValidGitDir(path):
723     return git_dir(path) != None
724
725 def parseRevision(ref):
726     return read_pipe("git rev-parse %s" % ref).strip()
727
728 def branchExists(ref):
729     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
730                      ignore_error=True)
731     return len(rev) > 0
732
733 def extractLogMessageFromGitCommit(commit):
734     logMessage = ""
735
736     ## fixme: title is first line of commit, not 1st paragraph.
737     foundTitle = False
738     for log in read_pipe_lines("git cat-file commit %s" % commit):
739        if not foundTitle:
740            if len(log) == 1:
741                foundTitle = True
742            continue
743
744        logMessage += log
745     return logMessage
746
747 def extractSettingsGitLog(log):
748     values = {}
749     for line in log.split("\n"):
750         line = line.strip()
751         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
752         if not m:
753             continue
754
755         assignments = m.group(1).split (':')
756         for a in assignments:
757             vals = a.split ('=')
758             key = vals[0].strip()
759             val = ('='.join (vals[1:])).strip()
760             if val.endswith ('\"') and val.startswith('"'):
761                 val = val[1:-1]
762
763             values[key] = val
764
765     paths = values.get("depot-paths")
766     if not paths:
767         paths = values.get("depot-path")
768     if paths:
769         values['depot-paths'] = paths.split(',')
770     return values
771
772 def gitBranchExists(branch):
773     proc = subprocess.Popen(["git", "rev-parse", branch],
774                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
775     return proc.wait() == 0;
776
777 def gitUpdateRef(ref, newvalue):
778     subprocess.check_call(["git", "update-ref", ref, newvalue])
779
780 def gitDeleteRef(ref):
781     subprocess.check_call(["git", "update-ref", "-d", ref])
782
783 _gitConfig = {}
784
785 def gitConfig(key, typeSpecifier=None):
786     if key not in _gitConfig:
787         cmd = [ "git", "config" ]
788         if typeSpecifier:
789             cmd += [ typeSpecifier ]
790         cmd += [ key ]
791         s = read_pipe(cmd, ignore_error=True)
792         _gitConfig[key] = s.strip()
793     return _gitConfig[key]
794
795 def gitConfigBool(key):
796     """Return a bool, using git config --bool.  It is True only if the
797        variable is set to true, and False if set to false or not present
798        in the config."""
799
800     if key not in _gitConfig:
801         _gitConfig[key] = gitConfig(key, '--bool') == "true"
802     return _gitConfig[key]
803
804 def gitConfigInt(key):
805     if key not in _gitConfig:
806         cmd = [ "git", "config", "--int", key ]
807         s = read_pipe(cmd, ignore_error=True)
808         v = s.strip()
809         try:
810             _gitConfig[key] = int(gitConfig(key, '--int'))
811         except ValueError:
812             _gitConfig[key] = None
813     return _gitConfig[key]
814
815 def gitConfigList(key):
816     if key not in _gitConfig:
817         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
818         _gitConfig[key] = s.strip().splitlines()
819         if _gitConfig[key] == ['']:
820             _gitConfig[key] = []
821     return _gitConfig[key]
822
823 def p4BranchesInGit(branchesAreInRemotes=True):
824     """Find all the branches whose names start with "p4/", looking
825        in remotes or heads as specified by the argument.  Return
826        a dictionary of { branch: revision } for each one found.
827        The branch names are the short names, without any
828        "p4/" prefix."""
829
830     branches = {}
831
832     cmdline = "git rev-parse --symbolic "
833     if branchesAreInRemotes:
834         cmdline += "--remotes"
835     else:
836         cmdline += "--branches"
837
838     for line in read_pipe_lines(cmdline):
839         line = line.strip()
840
841         # only import to p4/
842         if not line.startswith('p4/'):
843             continue
844         # special symbolic ref to p4/master
845         if line == "p4/HEAD":
846             continue
847
848         # strip off p4/ prefix
849         branch = line[len("p4/"):]
850
851         branches[branch] = parseRevision(line)
852
853     return branches
854
855 def branch_exists(branch):
856     """Make sure that the given ref name really exists."""
857
858     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
859     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
860     out, _ = p.communicate()
861     if p.returncode:
862         return False
863     # expect exactly one line of output: the branch name
864     return out.rstrip() == branch
865
866 def findUpstreamBranchPoint(head = "HEAD"):
867     branches = p4BranchesInGit()
868     # map from depot-path to branch name
869     branchByDepotPath = {}
870     for branch in branches.keys():
871         tip = branches[branch]
872         log = extractLogMessageFromGitCommit(tip)
873         settings = extractSettingsGitLog(log)
874         if "depot-paths" in settings:
875             paths = ",".join(settings["depot-paths"])
876             branchByDepotPath[paths] = "remotes/p4/" + branch
877
878     settings = None
879     parent = 0
880     while parent < 65535:
881         commit = head + "~%s" % parent
882         log = extractLogMessageFromGitCommit(commit)
883         settings = extractSettingsGitLog(log)
884         if "depot-paths" in settings:
885             paths = ",".join(settings["depot-paths"])
886             if paths in branchByDepotPath:
887                 return [branchByDepotPath[paths], settings]
888
889         parent = parent + 1
890
891     return ["", settings]
892
893 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
894     if not silent:
895         print("Creating/updating branch(es) in %s based on origin branch(es)"
896                % localRefPrefix)
897
898     originPrefix = "origin/p4/"
899
900     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
901         line = line.strip()
902         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
903             continue
904
905         headName = line[len(originPrefix):]
906         remoteHead = localRefPrefix + headName
907         originHead = line
908
909         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
910         if ('depot-paths' not in original
911             or 'change' not in original):
912             continue
913
914         update = False
915         if not gitBranchExists(remoteHead):
916             if verbose:
917                 print("creating %s" % remoteHead)
918             update = True
919         else:
920             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
921             if 'change' in settings:
922                 if settings['depot-paths'] == original['depot-paths']:
923                     originP4Change = int(original['change'])
924                     p4Change = int(settings['change'])
925                     if originP4Change > p4Change:
926                         print("%s (%s) is newer than %s (%s). "
927                                "Updating p4 branch from origin."
928                                % (originHead, originP4Change,
929                                   remoteHead, p4Change))
930                         update = True
931                 else:
932                     print("Ignoring: %s was imported from %s while "
933                            "%s was imported from %s"
934                            % (originHead, ','.join(original['depot-paths']),
935                               remoteHead, ','.join(settings['depot-paths'])))
936
937         if update:
938             system("git update-ref %s %s" % (remoteHead, originHead))
939
940 def originP4BranchesExist():
941         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
942
943
944 def p4ParseNumericChangeRange(parts):
945     changeStart = int(parts[0][1:])
946     if parts[1] == '#head':
947         changeEnd = p4_last_change()
948     else:
949         changeEnd = int(parts[1])
950
951     return (changeStart, changeEnd)
952
953 def chooseBlockSize(blockSize):
954     if blockSize:
955         return blockSize
956     else:
957         return defaultBlockSize
958
959 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
960     assert depotPaths
961
962     # Parse the change range into start and end. Try to find integer
963     # revision ranges as these can be broken up into blocks to avoid
964     # hitting server-side limits (maxrows, maxscanresults). But if
965     # that doesn't work, fall back to using the raw revision specifier
966     # strings, without using block mode.
967
968     if changeRange is None or changeRange == '':
969         changeStart = 1
970         changeEnd = p4_last_change()
971         block_size = chooseBlockSize(requestedBlockSize)
972     else:
973         parts = changeRange.split(',')
974         assert len(parts) == 2
975         try:
976             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
977             block_size = chooseBlockSize(requestedBlockSize)
978         except ValueError:
979             changeStart = parts[0][1:]
980             changeEnd = parts[1]
981             if requestedBlockSize:
982                 die("cannot use --changes-block-size with non-numeric revisions")
983             block_size = None
984
985     changes = set()
986
987     # Retrieve changes a block at a time, to prevent running
988     # into a MaxResults/MaxScanRows error from the server. If
989     # we _do_ hit one of those errors, turn down the block size
990
991     while True:
992         cmd = ['changes']
993
994         if block_size:
995             end = min(changeEnd, changeStart + block_size)
996             revisionRange = "%d,%d" % (changeStart, end)
997         else:
998             revisionRange = "%s,%s" % (changeStart, changeEnd)
999
1000         for p in depotPaths:
1001             cmd += ["%s...@%s" % (p, revisionRange)]
1002
1003         # fetch the changes
1004         try:
1005             result = p4CmdList(cmd, errors_as_exceptions=True)
1006         except P4RequestSizeException as e:
1007             if not block_size:
1008                 block_size = e.limit
1009             elif block_size > e.limit:
1010                 block_size = e.limit
1011             else:
1012                 block_size = max(2, block_size // 2)
1013
1014             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1015             continue
1016         except P4Exception as e:
1017             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1018
1019         # Insert changes in chronological order
1020         for entry in reversed(result):
1021             if 'change' not in entry:
1022                 continue
1023             changes.add(int(entry['change']))
1024
1025         if not block_size:
1026             break
1027
1028         if end >= changeEnd:
1029             break
1030
1031         changeStart = end + 1
1032
1033     changes = sorted(changes)
1034     return changes
1035
1036 def p4PathStartsWith(path, prefix):
1037     # This method tries to remedy a potential mixed-case issue:
1038     #
1039     # If UserA adds  //depot/DirA/file1
1040     # and UserB adds //depot/dira/file2
1041     #
1042     # we may or may not have a problem. If you have core.ignorecase=true,
1043     # we treat DirA and dira as the same directory
1044     if gitConfigBool("core.ignorecase"):
1045         return path.lower().startswith(prefix.lower())
1046     return path.startswith(prefix)
1047
1048 def getClientSpec():
1049     """Look at the p4 client spec, create a View() object that contains
1050        all the mappings, and return it."""
1051
1052     specList = p4CmdList("client -o")
1053     if len(specList) != 1:
1054         die('Output from "client -o" is %d lines, expecting 1' %
1055             len(specList))
1056
1057     # dictionary of all client parameters
1058     entry = specList[0]
1059
1060     # the //client/ name
1061     client_name = entry["Client"]
1062
1063     # just the keys that start with "View"
1064     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1065
1066     # hold this new View
1067     view = View(client_name)
1068
1069     # append the lines, in order, to the view
1070     for view_num in range(len(view_keys)):
1071         k = "View%d" % view_num
1072         if k not in view_keys:
1073             die("Expected view key %s missing" % k)
1074         view.append(entry[k])
1075
1076     return view
1077
1078 def getClientRoot():
1079     """Grab the client directory."""
1080
1081     output = p4CmdList("client -o")
1082     if len(output) != 1:
1083         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1084
1085     entry = output[0]
1086     if "Root" not in entry:
1087         die('Client has no "Root"')
1088
1089     return entry["Root"]
1090
1091 #
1092 # P4 wildcards are not allowed in filenames.  P4 complains
1093 # if you simply add them, but you can force it with "-f", in
1094 # which case it translates them into %xx encoding internally.
1095 #
1096 def wildcard_decode(path):
1097     # Search for and fix just these four characters.  Do % last so
1098     # that fixing it does not inadvertently create new %-escapes.
1099     # Cannot have * in a filename in windows; untested as to
1100     # what p4 would do in such a case.
1101     if not platform.system() == "Windows":
1102         path = path.replace("%2A", "*")
1103     path = path.replace("%23", "#") \
1104                .replace("%40", "@") \
1105                .replace("%25", "%")
1106     return path
1107
1108 def wildcard_encode(path):
1109     # do % first to avoid double-encoding the %s introduced here
1110     path = path.replace("%", "%25") \
1111                .replace("*", "%2A") \
1112                .replace("#", "%23") \
1113                .replace("@", "%40")
1114     return path
1115
1116 def wildcard_present(path):
1117     m = re.search("[*#@%]", path)
1118     return m is not None
1119
1120 class LargeFileSystem(object):
1121     """Base class for large file system support."""
1122
1123     def __init__(self, writeToGitStream):
1124         self.largeFiles = set()
1125         self.writeToGitStream = writeToGitStream
1126
1127     def generatePointer(self, cloneDestination, contentFile):
1128         """Return the content of a pointer file that is stored in Git instead of
1129            the actual content."""
1130         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1131
1132     def pushFile(self, localLargeFile):
1133         """Push the actual content which is not stored in the Git repository to
1134            a server."""
1135         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1136
1137     def hasLargeFileExtension(self, relPath):
1138         return reduce(
1139             lambda a, b: a or b,
1140             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1141             False
1142         )
1143
1144     def generateTempFile(self, contents):
1145         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146         for d in contents:
1147             contentFile.write(d)
1148         contentFile.close()
1149         return contentFile.name
1150
1151     def exceedsLargeFileThreshold(self, relPath, contents):
1152         if gitConfigInt('git-p4.largeFileThreshold'):
1153             contentsSize = sum(len(d) for d in contents)
1154             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1155                 return True
1156         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1157             contentsSize = sum(len(d) for d in contents)
1158             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159                 return False
1160             contentTempFile = self.generateTempFile(contents)
1161             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1162             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1163             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1164             zf.close()
1165             compressedContentsSize = zf.infolist()[0].compress_size
1166             os.remove(contentTempFile)
1167             os.remove(compressedContentFile.name)
1168             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169                 return True
1170         return False
1171
1172     def addLargeFile(self, relPath):
1173         self.largeFiles.add(relPath)
1174
1175     def removeLargeFile(self, relPath):
1176         self.largeFiles.remove(relPath)
1177
1178     def isLargeFile(self, relPath):
1179         return relPath in self.largeFiles
1180
1181     def processContent(self, git_mode, relPath, contents):
1182         """Processes the content of git fast import. This method decides if a
1183            file is stored in the large file system and handles all necessary
1184            steps."""
1185         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186             contentTempFile = self.generateTempFile(contents)
1187             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188             if pointer_git_mode:
1189                 git_mode = pointer_git_mode
1190             if localLargeFile:
1191                 # Move temp file to final location in large file system
1192                 largeFileDir = os.path.dirname(localLargeFile)
1193                 if not os.path.isdir(largeFileDir):
1194                     os.makedirs(largeFileDir)
1195                 shutil.move(contentTempFile, localLargeFile)
1196                 self.addLargeFile(relPath)
1197                 if gitConfigBool('git-p4.largeFilePush'):
1198                     self.pushFile(localLargeFile)
1199                 if verbose:
1200                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201         return (git_mode, contents)
1202
1203 class MockLFS(LargeFileSystem):
1204     """Mock large file system for testing."""
1205
1206     def generatePointer(self, contentFile):
1207         """The pointer content is the original content prefixed with "pointer-".
1208            The local filename of the large file storage is derived from the file content.
1209            """
1210         with open(contentFile, 'r') as f:
1211             content = next(f)
1212             gitMode = '100644'
1213             pointerContents = 'pointer-' + content
1214             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215             return (gitMode, pointerContents, localLargeFile)
1216
1217     def pushFile(self, localLargeFile):
1218         """The remote filename of the large file storage is the same as the local
1219            one but in a different directory.
1220            """
1221         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222         if not os.path.exists(remotePath):
1223             os.makedirs(remotePath)
1224         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1225
1226 class GitLFS(LargeFileSystem):
1227     """Git LFS as backend for the git-p4 large file system.
1228        See https://git-lfs.github.com/ for details."""
1229
1230     def __init__(self, *args):
1231         LargeFileSystem.__init__(self, *args)
1232         self.baseGitAttributes = []
1233
1234     def generatePointer(self, contentFile):
1235         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236            mode and content which is stored in the Git repository instead of
1237            the actual content. Return also the new location of the actual
1238            content.
1239            """
1240         if os.path.getsize(contentFile) == 0:
1241             return (None, '', None)
1242
1243         pointerProcess = subprocess.Popen(
1244             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245             stdout=subprocess.PIPE
1246         )
1247         pointerFile = pointerProcess.stdout.read()
1248         if pointerProcess.wait():
1249             os.remove(contentFile)
1250             die('git-lfs pointer command failed. Did you install the extension?')
1251
1252         # Git LFS removed the preamble in the output of the 'pointer' command
1253         # starting from version 1.2.0. Check for the preamble here to support
1254         # earlier versions.
1255         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256         if pointerFile.startswith('Git LFS pointer for'):
1257             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1258
1259         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260         localLargeFile = os.path.join(
1261             os.getcwd(),
1262             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1263             oid,
1264         )
1265         # LFS Spec states that pointer files should not have the executable bit set.
1266         gitMode = '100644'
1267         return (gitMode, pointerFile, localLargeFile)
1268
1269     def pushFile(self, localLargeFile):
1270         uploadProcess = subprocess.Popen(
1271             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1272         )
1273         if uploadProcess.wait():
1274             die('git-lfs push command failed. Did you define a remote?')
1275
1276     def generateGitAttributes(self):
1277         return (
1278             self.baseGitAttributes +
1279             [
1280                 '\n',
1281                 '#\n',
1282                 '# Git LFS (see https://git-lfs.github.com/)\n',
1283                 '#\n',
1284             ] +
1285             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1286                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1287             ] +
1288             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1289                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1290             ]
1291         )
1292
1293     def addLargeFile(self, relPath):
1294         LargeFileSystem.addLargeFile(self, relPath)
1295         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1296
1297     def removeLargeFile(self, relPath):
1298         LargeFileSystem.removeLargeFile(self, relPath)
1299         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1300
1301     def processContent(self, git_mode, relPath, contents):
1302         if relPath == '.gitattributes':
1303             self.baseGitAttributes = contents
1304             return (git_mode, self.generateGitAttributes())
1305         else:
1306             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1307
1308 class Command:
1309     delete_actions = ( "delete", "move/delete", "purge" )
1310     add_actions = ( "add", "move/add" )
1311
1312     def __init__(self):
1313         self.usage = "usage: %prog [options]"
1314         self.needsGit = True
1315         self.verbose = False
1316
1317     # This is required for the "append" cloneExclude action
1318     def ensure_value(self, attr, value):
1319         if not hasattr(self, attr) or getattr(self, attr) is None:
1320             setattr(self, attr, value)
1321         return getattr(self, attr)
1322
1323 class P4UserMap:
1324     def __init__(self):
1325         self.userMapFromPerforceServer = False
1326         self.myP4UserId = None
1327
1328     def p4UserId(self):
1329         if self.myP4UserId:
1330             return self.myP4UserId
1331
1332         results = p4CmdList("user -o")
1333         for r in results:
1334             if 'User' in r:
1335                 self.myP4UserId = r['User']
1336                 return r['User']
1337         die("Could not find your p4 user id")
1338
1339     def p4UserIsMe(self, p4User):
1340         # return True if the given p4 user is actually me
1341         me = self.p4UserId()
1342         if not p4User or p4User != me:
1343             return False
1344         else:
1345             return True
1346
1347     def getUserCacheFilename(self):
1348         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1349         return home + "/.gitp4-usercache.txt"
1350
1351     def getUserMapFromPerforceServer(self):
1352         if self.userMapFromPerforceServer:
1353             return
1354         self.users = {}
1355         self.emails = {}
1356
1357         for output in p4CmdList("users"):
1358             if "User" not in output:
1359                 continue
1360             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1361             self.emails[output["Email"]] = output["User"]
1362
1363         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1364         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1365             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1366             if mapUser and len(mapUser[0]) == 3:
1367                 user = mapUser[0][0]
1368                 fullname = mapUser[0][1]
1369                 email = mapUser[0][2]
1370                 self.users[user] = fullname + " <" + email + ">"
1371                 self.emails[email] = user
1372
1373         s = ''
1374         for (key, val) in self.users.items():
1375             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1376
1377         open(self.getUserCacheFilename(), "wb").write(s)
1378         self.userMapFromPerforceServer = True
1379
1380     def loadUserMapFromCache(self):
1381         self.users = {}
1382         self.userMapFromPerforceServer = False
1383         try:
1384             cache = open(self.getUserCacheFilename(), "rb")
1385             lines = cache.readlines()
1386             cache.close()
1387             for line in lines:
1388                 entry = line.strip().split("\t")
1389                 self.users[entry[0]] = entry[1]
1390         except IOError:
1391             self.getUserMapFromPerforceServer()
1392
1393 class P4Debug(Command):
1394     def __init__(self):
1395         Command.__init__(self)
1396         self.options = []
1397         self.description = "A tool to debug the output of p4 -G."
1398         self.needsGit = False
1399
1400     def run(self, args):
1401         j = 0
1402         for output in p4CmdList(args):
1403             print('Element: %d' % j)
1404             j += 1
1405             print(output)
1406         return True
1407
1408 class P4RollBack(Command):
1409     def __init__(self):
1410         Command.__init__(self)
1411         self.options = [
1412             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1413         ]
1414         self.description = "A tool to debug the multi-branch import. Don't use :)"
1415         self.rollbackLocalBranches = False
1416
1417     def run(self, args):
1418         if len(args) != 1:
1419             return False
1420         maxChange = int(args[0])
1421
1422         if "p4ExitCode" in p4Cmd("changes -m 1"):
1423             die("Problems executing p4");
1424
1425         if self.rollbackLocalBranches:
1426             refPrefix = "refs/heads/"
1427             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1428         else:
1429             refPrefix = "refs/remotes/"
1430             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1431
1432         for line in lines:
1433             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1434                 line = line.strip()
1435                 ref = refPrefix + line
1436                 log = extractLogMessageFromGitCommit(ref)
1437                 settings = extractSettingsGitLog(log)
1438
1439                 depotPaths = settings['depot-paths']
1440                 change = settings['change']
1441
1442                 changed = False
1443
1444                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1445                                                            for p in depotPaths]))) == 0:
1446                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1447                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1448                     continue
1449
1450                 while change and int(change) > maxChange:
1451                     changed = True
1452                     if self.verbose:
1453                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1454                     system("git update-ref %s \"%s^\"" % (ref, ref))
1455                     log = extractLogMessageFromGitCommit(ref)
1456                     settings =  extractSettingsGitLog(log)
1457
1458
1459                     depotPaths = settings['depot-paths']
1460                     change = settings['change']
1461
1462                 if changed:
1463                     print("%s rewound to %s" % (ref, change))
1464
1465         return True
1466
1467 class P4Submit(Command, P4UserMap):
1468
1469     conflict_behavior_choices = ("ask", "skip", "quit")
1470
1471     def __init__(self):
1472         Command.__init__(self)
1473         P4UserMap.__init__(self)
1474         self.options = [
1475                 optparse.make_option("--origin", dest="origin"),
1476                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1477                 # preserve the user, requires relevant p4 permissions
1478                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1479                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1480                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1481                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1482                 optparse.make_option("--conflict", dest="conflict_behavior",
1483                                      choices=self.conflict_behavior_choices),
1484                 optparse.make_option("--branch", dest="branch"),
1485                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1486                                      help="Shelve instead of submit. Shelved files are reverted, "
1487                                      "restoring the workspace to the state before the shelve"),
1488                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1489                                      metavar="CHANGELIST",
1490                                      help="update an existing shelved changelist, implies --shelve, "
1491                                            "repeat in-order for multiple shelved changelists"),
1492                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1493                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1494                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1495                                      help="Disable rebase after submit is completed. Can be useful if you "
1496                                      "work from a local git branch that is not master"),
1497                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1498                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1499         ]
1500         self.description = """Submit changes from git to the perforce depot.\n
1501     The `p4-pre-submit` hook is executed if it exists and is executable.
1502     The hook takes no parameters and nothing from standard input. Exiting with
1503     non-zero status from this script prevents `git-p4 submit` from launching.
1504
1505     One usage scenario is to run unit tests in the hook."""
1506
1507         self.usage += " [name of git branch to submit into perforce depot]"
1508         self.origin = ""
1509         self.detectRenames = False
1510         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1511         self.dry_run = False
1512         self.shelve = False
1513         self.update_shelve = list()
1514         self.commit = ""
1515         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1516         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1517         self.prepare_p4_only = False
1518         self.conflict_behavior = None
1519         self.isWindows = (platform.system() == "Windows")
1520         self.exportLabels = False
1521         self.p4HasMoveCommand = p4_has_move_command()
1522         self.branch = None
1523
1524         if gitConfig('git-p4.largeFileSystem'):
1525             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1526
1527     def check(self):
1528         if len(p4CmdList("opened ...")) > 0:
1529             die("You have files opened with perforce! Close them before starting the sync.")
1530
1531     def separate_jobs_from_description(self, message):
1532         """Extract and return a possible Jobs field in the commit
1533            message.  It goes into a separate section in the p4 change
1534            specification.
1535
1536            A jobs line starts with "Jobs:" and looks like a new field
1537            in a form.  Values are white-space separated on the same
1538            line or on following lines that start with a tab.
1539
1540            This does not parse and extract the full git commit message
1541            like a p4 form.  It just sees the Jobs: line as a marker
1542            to pass everything from then on directly into the p4 form,
1543            but outside the description section.
1544
1545            Return a tuple (stripped log message, jobs string)."""
1546
1547         m = re.search(r'^Jobs:', message, re.MULTILINE)
1548         if m is None:
1549             return (message, None)
1550
1551         jobtext = message[m.start():]
1552         stripped_message = message[:m.start()].rstrip()
1553         return (stripped_message, jobtext)
1554
1555     def prepareLogMessage(self, template, message, jobs):
1556         """Edits the template returned from "p4 change -o" to insert
1557            the message in the Description field, and the jobs text in
1558            the Jobs field."""
1559         result = ""
1560
1561         inDescriptionSection = False
1562
1563         for line in template.split("\n"):
1564             if line.startswith("#"):
1565                 result += line + "\n"
1566                 continue
1567
1568             if inDescriptionSection:
1569                 if line.startswith("Files:") or line.startswith("Jobs:"):
1570                     inDescriptionSection = False
1571                     # insert Jobs section
1572                     if jobs:
1573                         result += jobs + "\n"
1574                 else:
1575                     continue
1576             else:
1577                 if line.startswith("Description:"):
1578                     inDescriptionSection = True
1579                     line += "\n"
1580                     for messageLine in message.split("\n"):
1581                         line += "\t" + messageLine + "\n"
1582
1583             result += line + "\n"
1584
1585         return result
1586
1587     def patchRCSKeywords(self, file, pattern):
1588         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1589         (handle, outFileName) = tempfile.mkstemp(dir='.')
1590         try:
1591             outFile = os.fdopen(handle, "w+")
1592             inFile = open(file, "r")
1593             regexp = re.compile(pattern, re.VERBOSE)
1594             for line in inFile.readlines():
1595                 line = regexp.sub(r'$\1$', line)
1596                 outFile.write(line)
1597             inFile.close()
1598             outFile.close()
1599             # Forcibly overwrite the original file
1600             os.unlink(file)
1601             shutil.move(outFileName, file)
1602         except:
1603             # cleanup our temporary file
1604             os.unlink(outFileName)
1605             print("Failed to strip RCS keywords in %s" % file)
1606             raise
1607
1608         print("Patched up RCS keywords in %s" % file)
1609
1610     def p4UserForCommit(self,id):
1611         # Return the tuple (perforce user,git email) for a given git commit id
1612         self.getUserMapFromPerforceServer()
1613         gitEmail = read_pipe(["git", "log", "--max-count=1",
1614                               "--format=%ae", id])
1615         gitEmail = gitEmail.strip()
1616         if gitEmail not in self.emails:
1617             return (None,gitEmail)
1618         else:
1619             return (self.emails[gitEmail],gitEmail)
1620
1621     def checkValidP4Users(self,commits):
1622         # check if any git authors cannot be mapped to p4 users
1623         for id in commits:
1624             (user,email) = self.p4UserForCommit(id)
1625             if not user:
1626                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1627                 if gitConfigBool("git-p4.allowMissingP4Users"):
1628                     print("%s" % msg)
1629                 else:
1630                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1631
1632     def lastP4Changelist(self):
1633         # Get back the last changelist number submitted in this client spec. This
1634         # then gets used to patch up the username in the change. If the same
1635         # client spec is being used by multiple processes then this might go
1636         # wrong.
1637         results = p4CmdList("client -o")        # find the current client
1638         client = None
1639         for r in results:
1640             if 'Client' in r:
1641                 client = r['Client']
1642                 break
1643         if not client:
1644             die("could not get client spec")
1645         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1646         for r in results:
1647             if 'change' in r:
1648                 return r['change']
1649         die("Could not get changelist number for last submit - cannot patch up user details")
1650
1651     def modifyChangelistUser(self, changelist, newUser):
1652         # fixup the user field of a changelist after it has been submitted.
1653         changes = p4CmdList("change -o %s" % changelist)
1654         if len(changes) != 1:
1655             die("Bad output from p4 change modifying %s to user %s" %
1656                 (changelist, newUser))
1657
1658         c = changes[0]
1659         if c['User'] == newUser: return   # nothing to do
1660         c['User'] = newUser
1661         input = marshal.dumps(c)
1662
1663         result = p4CmdList("change -f -i", stdin=input)
1664         for r in result:
1665             if 'code' in r:
1666                 if r['code'] == 'error':
1667                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1668             if 'data' in r:
1669                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1670                 return
1671         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1672
1673     def canChangeChangelists(self):
1674         # check to see if we have p4 admin or super-user permissions, either of
1675         # which are required to modify changelists.
1676         results = p4CmdList(["protects", self.depotPath])
1677         for r in results:
1678             if 'perm' in r:
1679                 if r['perm'] == 'admin':
1680                     return 1
1681                 if r['perm'] == 'super':
1682                     return 1
1683         return 0
1684
1685     def prepareSubmitTemplate(self, changelist=None):
1686         """Run "p4 change -o" to grab a change specification template.
1687            This does not use "p4 -G", as it is nice to keep the submission
1688            template in original order, since a human might edit it.
1689
1690            Remove lines in the Files section that show changes to files
1691            outside the depot path we're committing into."""
1692
1693         [upstream, settings] = findUpstreamBranchPoint()
1694
1695         template = """\
1696 # A Perforce Change Specification.
1697 #
1698 #  Change:      The change number. 'new' on a new changelist.
1699 #  Date:        The date this specification was last modified.
1700 #  Client:      The client on which the changelist was created.  Read-only.
1701 #  User:        The user who created the changelist.
1702 #  Status:      Either 'pending' or 'submitted'. Read-only.
1703 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1704 #  Description: Comments about the changelist.  Required.
1705 #  Jobs:        What opened jobs are to be closed by this changelist.
1706 #               You may delete jobs from this list.  (New changelists only.)
1707 #  Files:       What opened files from the default changelist are to be added
1708 #               to this changelist.  You may delete files from this list.
1709 #               (New changelists only.)
1710 """
1711         files_list = []
1712         inFilesSection = False
1713         change_entry = None
1714         args = ['change', '-o']
1715         if changelist:
1716             args.append(str(changelist))
1717         for entry in p4CmdList(args):
1718             if 'code' not in entry:
1719                 continue
1720             if entry['code'] == 'stat':
1721                 change_entry = entry
1722                 break
1723         if not change_entry:
1724             die('Failed to decode output of p4 change -o')
1725         for key, value in change_entry.iteritems():
1726             if key.startswith('File'):
1727                 if 'depot-paths' in settings:
1728                     if not [p for p in settings['depot-paths']
1729                             if p4PathStartsWith(value, p)]:
1730                         continue
1731                 else:
1732                     if not p4PathStartsWith(value, self.depotPath):
1733                         continue
1734                 files_list.append(value)
1735                 continue
1736         # Output in the order expected by prepareLogMessage
1737         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1738             if key not in change_entry:
1739                 continue
1740             template += '\n'
1741             template += key + ':'
1742             if key == 'Description':
1743                 template += '\n'
1744             for field_line in change_entry[key].splitlines():
1745                 template += '\t'+field_line+'\n'
1746         if len(files_list) > 0:
1747             template += '\n'
1748             template += 'Files:\n'
1749         for path in files_list:
1750             template += '\t'+path+'\n'
1751         return template
1752
1753     def edit_template(self, template_file):
1754         """Invoke the editor to let the user change the submission
1755            message.  Return true if okay to continue with the submit."""
1756
1757         # if configured to skip the editing part, just submit
1758         if gitConfigBool("git-p4.skipSubmitEdit"):
1759             return True
1760
1761         # look at the modification time, to check later if the user saved
1762         # the file
1763         mtime = os.stat(template_file).st_mtime
1764
1765         # invoke the editor
1766         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1767             editor = os.environ.get("P4EDITOR")
1768         else:
1769             editor = read_pipe("git var GIT_EDITOR").strip()
1770         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1771
1772         # If the file was not saved, prompt to see if this patch should
1773         # be skipped.  But skip this verification step if configured so.
1774         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1775             return True
1776
1777         # modification time updated means user saved the file
1778         if os.stat(template_file).st_mtime > mtime:
1779             return True
1780
1781         while True:
1782             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1783             if response == 'y':
1784                 return True
1785             if response == 'n':
1786                 return False
1787
1788     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1789         # diff
1790         if "P4DIFF" in os.environ:
1791             del(os.environ["P4DIFF"])
1792         diff = ""
1793         for editedFile in editedFiles:
1794             diff += p4_read_pipe(['diff', '-du',
1795                                   wildcard_encode(editedFile)])
1796
1797         # new file diff
1798         newdiff = ""
1799         for newFile in filesToAdd:
1800             newdiff += "==== new file ====\n"
1801             newdiff += "--- /dev/null\n"
1802             newdiff += "+++ %s\n" % newFile
1803
1804             is_link = os.path.islink(newFile)
1805             expect_link = newFile in symlinks
1806
1807             if is_link and expect_link:
1808                 newdiff += "+%s\n" % os.readlink(newFile)
1809             else:
1810                 f = open(newFile, "r")
1811                 for line in f.readlines():
1812                     newdiff += "+" + line
1813                 f.close()
1814
1815         return (diff + newdiff).replace('\r\n', '\n')
1816
1817     def applyCommit(self, id):
1818         """Apply one commit, return True if it succeeded."""
1819
1820         print("Applying", read_pipe(["git", "show", "-s",
1821                                      "--format=format:%h %s", id]))
1822
1823         (p4User, gitEmail) = self.p4UserForCommit(id)
1824
1825         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1826         filesToAdd = set()
1827         filesToChangeType = set()
1828         filesToDelete = set()
1829         editedFiles = set()
1830         pureRenameCopy = set()
1831         symlinks = set()
1832         filesToChangeExecBit = {}
1833         all_files = list()
1834
1835         for line in diff:
1836             diff = parseDiffTreeEntry(line)
1837             modifier = diff['status']
1838             path = diff['src']
1839             all_files.append(path)
1840
1841             if modifier == "M":
1842                 p4_edit(path)
1843                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1844                     filesToChangeExecBit[path] = diff['dst_mode']
1845                 editedFiles.add(path)
1846             elif modifier == "A":
1847                 filesToAdd.add(path)
1848                 filesToChangeExecBit[path] = diff['dst_mode']
1849                 if path in filesToDelete:
1850                     filesToDelete.remove(path)
1851
1852                 dst_mode = int(diff['dst_mode'], 8)
1853                 if dst_mode == 0o120000:
1854                     symlinks.add(path)
1855
1856             elif modifier == "D":
1857                 filesToDelete.add(path)
1858                 if path in filesToAdd:
1859                     filesToAdd.remove(path)
1860             elif modifier == "C":
1861                 src, dest = diff['src'], diff['dst']
1862                 p4_integrate(src, dest)
1863                 pureRenameCopy.add(dest)
1864                 if diff['src_sha1'] != diff['dst_sha1']:
1865                     p4_edit(dest)
1866                     pureRenameCopy.discard(dest)
1867                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1868                     p4_edit(dest)
1869                     pureRenameCopy.discard(dest)
1870                     filesToChangeExecBit[dest] = diff['dst_mode']
1871                 if self.isWindows:
1872                     # turn off read-only attribute
1873                     os.chmod(dest, stat.S_IWRITE)
1874                 os.unlink(dest)
1875                 editedFiles.add(dest)
1876             elif modifier == "R":
1877                 src, dest = diff['src'], diff['dst']
1878                 if self.p4HasMoveCommand:
1879                     p4_edit(src)        # src must be open before move
1880                     p4_move(src, dest)  # opens for (move/delete, move/add)
1881                 else:
1882                     p4_integrate(src, dest)
1883                     if diff['src_sha1'] != diff['dst_sha1']:
1884                         p4_edit(dest)
1885                     else:
1886                         pureRenameCopy.add(dest)
1887                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1888                     if not self.p4HasMoveCommand:
1889                         p4_edit(dest)   # with move: already open, writable
1890                     filesToChangeExecBit[dest] = diff['dst_mode']
1891                 if not self.p4HasMoveCommand:
1892                     if self.isWindows:
1893                         os.chmod(dest, stat.S_IWRITE)
1894                     os.unlink(dest)
1895                     filesToDelete.add(src)
1896                 editedFiles.add(dest)
1897             elif modifier == "T":
1898                 filesToChangeType.add(path)
1899             else:
1900                 die("unknown modifier %s for %s" % (modifier, path))
1901
1902         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1903         patchcmd = diffcmd + " | git apply "
1904         tryPatchCmd = patchcmd + "--check -"
1905         applyPatchCmd = patchcmd + "--check --apply -"
1906         patch_succeeded = True
1907
1908         if os.system(tryPatchCmd) != 0:
1909             fixed_rcs_keywords = False
1910             patch_succeeded = False
1911             print("Unfortunately applying the change failed!")
1912
1913             # Patch failed, maybe it's just RCS keyword woes. Look through
1914             # the patch to see if that's possible.
1915             if gitConfigBool("git-p4.attemptRCSCleanup"):
1916                 file = None
1917                 pattern = None
1918                 kwfiles = {}
1919                 for file in editedFiles | filesToDelete:
1920                     # did this file's delta contain RCS keywords?
1921                     pattern = p4_keywords_regexp_for_file(file)
1922
1923                     if pattern:
1924                         # this file is a possibility...look for RCS keywords.
1925                         regexp = re.compile(pattern, re.VERBOSE)
1926                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1927                             if regexp.search(line):
1928                                 if verbose:
1929                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1930                                 kwfiles[file] = pattern
1931                                 break
1932
1933                 for file in kwfiles:
1934                     if verbose:
1935                         print("zapping %s with %s" % (line,pattern))
1936                     # File is being deleted, so not open in p4.  Must
1937                     # disable the read-only bit on windows.
1938                     if self.isWindows and file not in editedFiles:
1939                         os.chmod(file, stat.S_IWRITE)
1940                     self.patchRCSKeywords(file, kwfiles[file])
1941                     fixed_rcs_keywords = True
1942
1943             if fixed_rcs_keywords:
1944                 print("Retrying the patch with RCS keywords cleaned up")
1945                 if os.system(tryPatchCmd) == 0:
1946                     patch_succeeded = True
1947
1948         if not patch_succeeded:
1949             for f in editedFiles:
1950                 p4_revert(f)
1951             return False
1952
1953         #
1954         # Apply the patch for real, and do add/delete/+x handling.
1955         #
1956         system(applyPatchCmd)
1957
1958         for f in filesToChangeType:
1959             p4_edit(f, "-t", "auto")
1960         for f in filesToAdd:
1961             p4_add(f)
1962         for f in filesToDelete:
1963             p4_revert(f)
1964             p4_delete(f)
1965
1966         # Set/clear executable bits
1967         for f in filesToChangeExecBit.keys():
1968             mode = filesToChangeExecBit[f]
1969             setP4ExecBit(f, mode)
1970
1971         update_shelve = 0
1972         if len(self.update_shelve) > 0:
1973             update_shelve = self.update_shelve.pop(0)
1974             p4_reopen_in_change(update_shelve, all_files)
1975
1976         #
1977         # Build p4 change description, starting with the contents
1978         # of the git commit message.
1979         #
1980         logMessage = extractLogMessageFromGitCommit(id)
1981         logMessage = logMessage.strip()
1982         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1983
1984         template = self.prepareSubmitTemplate(update_shelve)
1985         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1986
1987         if self.preserveUser:
1988            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1989
1990         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1991             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1992             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1993             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1994
1995         separatorLine = "######## everything below this line is just the diff #######\n"
1996         if not self.prepare_p4_only:
1997             submitTemplate += separatorLine
1998             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1999
2000         (handle, fileName) = tempfile.mkstemp()
2001         tmpFile = os.fdopen(handle, "w+b")
2002         if self.isWindows:
2003             submitTemplate = submitTemplate.replace("\n", "\r\n")
2004         tmpFile.write(submitTemplate)
2005         tmpFile.close()
2006
2007         if self.prepare_p4_only:
2008             #
2009             # Leave the p4 tree prepared, and the submit template around
2010             # and let the user decide what to do next
2011             #
2012             print()
2013             print("P4 workspace prepared for submission.")
2014             print("To submit or revert, go to client workspace")
2015             print("  " + self.clientPath)
2016             print()
2017             print("To submit, use \"p4 submit\" to write a new description,")
2018             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2019                   " \"git p4\"." % fileName)
2020             print("You can delete the file \"%s\" when finished." % fileName)
2021
2022             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2023                 print("To preserve change ownership by user %s, you must\n" \
2024                       "do \"p4 change -f <change>\" after submitting and\n" \
2025                       "edit the User field.")
2026             if pureRenameCopy:
2027                 print("After submitting, renamed files must be re-synced.")
2028                 print("Invoke \"p4 sync -f\" on each of these files:")
2029                 for f in pureRenameCopy:
2030                     print("  " + f)
2031
2032             print()
2033             print("To revert the changes, use \"p4 revert ...\", and delete")
2034             print("the submit template file \"%s\"" % fileName)
2035             if filesToAdd:
2036                 print("Since the commit adds new files, they must be deleted:")
2037                 for f in filesToAdd:
2038                     print("  " + f)
2039             print()
2040             return True
2041
2042         #
2043         # Let the user edit the change description, then submit it.
2044         #
2045         submitted = False
2046
2047         try:
2048             if self.edit_template(fileName):
2049                 # read the edited message and submit
2050                 tmpFile = open(fileName, "rb")
2051                 message = tmpFile.read()
2052                 tmpFile.close()
2053                 if self.isWindows:
2054                     message = message.replace("\r\n", "\n")
2055                 submitTemplate = message[:message.index(separatorLine)]
2056
2057                 if update_shelve:
2058                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2059                 elif self.shelve:
2060                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2061                 else:
2062                     p4_write_pipe(['submit', '-i'], submitTemplate)
2063                     # The rename/copy happened by applying a patch that created a
2064                     # new file.  This leaves it writable, which confuses p4.
2065                     for f in pureRenameCopy:
2066                         p4_sync(f, "-f")
2067
2068                 if self.preserveUser:
2069                     if p4User:
2070                         # Get last changelist number. Cannot easily get it from
2071                         # the submit command output as the output is
2072                         # unmarshalled.
2073                         changelist = self.lastP4Changelist()
2074                         self.modifyChangelistUser(changelist, p4User)
2075
2076                 submitted = True
2077
2078         finally:
2079             # skip this patch
2080             if not submitted or self.shelve:
2081                 if self.shelve:
2082                     print ("Reverting shelved files.")
2083                 else:
2084                     print ("Submission cancelled, undoing p4 changes.")
2085                 for f in editedFiles | filesToDelete:
2086                     p4_revert(f)
2087                 for f in filesToAdd:
2088                     p4_revert(f)
2089                     os.remove(f)
2090
2091         os.remove(fileName)
2092         return submitted
2093
2094     # Export git tags as p4 labels. Create a p4 label and then tag
2095     # with that.
2096     def exportGitTags(self, gitTags):
2097         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2098         if len(validLabelRegexp) == 0:
2099             validLabelRegexp = defaultLabelRegexp
2100         m = re.compile(validLabelRegexp)
2101
2102         for name in gitTags:
2103
2104             if not m.match(name):
2105                 if verbose:
2106                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2107                 continue
2108
2109             # Get the p4 commit this corresponds to
2110             logMessage = extractLogMessageFromGitCommit(name)
2111             values = extractSettingsGitLog(logMessage)
2112
2113             if 'change' not in values:
2114                 # a tag pointing to something not sent to p4; ignore
2115                 if verbose:
2116                     print("git tag %s does not give a p4 commit" % name)
2117                 continue
2118             else:
2119                 changelist = values['change']
2120
2121             # Get the tag details.
2122             inHeader = True
2123             isAnnotated = False
2124             body = []
2125             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2126                 l = l.strip()
2127                 if inHeader:
2128                     if re.match(r'tag\s+', l):
2129                         isAnnotated = True
2130                     elif re.match(r'\s*$', l):
2131                         inHeader = False
2132                         continue
2133                 else:
2134                     body.append(l)
2135
2136             if not isAnnotated:
2137                 body = ["lightweight tag imported by git p4\n"]
2138
2139             # Create the label - use the same view as the client spec we are using
2140             clientSpec = getClientSpec()
2141
2142             labelTemplate  = "Label: %s\n" % name
2143             labelTemplate += "Description:\n"
2144             for b in body:
2145                 labelTemplate += "\t" + b + "\n"
2146             labelTemplate += "View:\n"
2147             for depot_side in clientSpec.mappings:
2148                 labelTemplate += "\t%s\n" % depot_side
2149
2150             if self.dry_run:
2151                 print("Would create p4 label %s for tag" % name)
2152             elif self.prepare_p4_only:
2153                 print("Not creating p4 label %s for tag due to option" \
2154                       " --prepare-p4-only" % name)
2155             else:
2156                 p4_write_pipe(["label", "-i"], labelTemplate)
2157
2158                 # Use the label
2159                 p4_system(["tag", "-l", name] +
2160                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2161
2162                 if verbose:
2163                     print("created p4 label for tag %s" % name)
2164
2165     def run(self, args):
2166         if len(args) == 0:
2167             self.master = currentGitBranch()
2168         elif len(args) == 1:
2169             self.master = args[0]
2170             if not branchExists(self.master):
2171                 die("Branch %s does not exist" % self.master)
2172         else:
2173             return False
2174
2175         for i in self.update_shelve:
2176             if i <= 0:
2177                 sys.exit("invalid changelist %d" % i)
2178
2179         if self.master:
2180             allowSubmit = gitConfig("git-p4.allowSubmit")
2181             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2182                 die("%s is not in git-p4.allowSubmit" % self.master)
2183
2184         [upstream, settings] = findUpstreamBranchPoint()
2185         self.depotPath = settings['depot-paths'][0]
2186         if len(self.origin) == 0:
2187             self.origin = upstream
2188
2189         if len(self.update_shelve) > 0:
2190             self.shelve = True
2191
2192         if self.preserveUser:
2193             if not self.canChangeChangelists():
2194                 die("Cannot preserve user names without p4 super-user or admin permissions")
2195
2196         # if not set from the command line, try the config file
2197         if self.conflict_behavior is None:
2198             val = gitConfig("git-p4.conflict")
2199             if val:
2200                 if val not in self.conflict_behavior_choices:
2201                     die("Invalid value '%s' for config git-p4.conflict" % val)
2202             else:
2203                 val = "ask"
2204             self.conflict_behavior = val
2205
2206         if self.verbose:
2207             print("Origin branch is " + self.origin)
2208
2209         if len(self.depotPath) == 0:
2210             print("Internal error: cannot locate perforce depot path from existing branches")
2211             sys.exit(128)
2212
2213         self.useClientSpec = False
2214         if gitConfigBool("git-p4.useclientspec"):
2215             self.useClientSpec = True
2216         if self.useClientSpec:
2217             self.clientSpecDirs = getClientSpec()
2218
2219         # Check for the existence of P4 branches
2220         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2221
2222         if self.useClientSpec and not branchesDetected:
2223             # all files are relative to the client spec
2224             self.clientPath = getClientRoot()
2225         else:
2226             self.clientPath = p4Where(self.depotPath)
2227
2228         if self.clientPath == "":
2229             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2230
2231         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2232         self.oldWorkingDirectory = os.getcwd()
2233
2234         # ensure the clientPath exists
2235         new_client_dir = False
2236         if not os.path.exists(self.clientPath):
2237             new_client_dir = True
2238             os.makedirs(self.clientPath)
2239
2240         chdir(self.clientPath, is_client_path=True)
2241         if self.dry_run:
2242             print("Would synchronize p4 checkout in %s" % self.clientPath)
2243         else:
2244             print("Synchronizing p4 checkout...")
2245             if new_client_dir:
2246                 # old one was destroyed, and maybe nobody told p4
2247                 p4_sync("...", "-f")
2248             else:
2249                 p4_sync("...")
2250         self.check()
2251
2252         commits = []
2253         if self.master:
2254             committish = self.master
2255         else:
2256             committish = 'HEAD'
2257
2258         if self.commit != "":
2259             if self.commit.find("..") != -1:
2260                 limits_ish = self.commit.split("..")
2261                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2262                     commits.append(line.strip())
2263                 commits.reverse()
2264             else:
2265                 commits.append(self.commit)
2266         else:
2267             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2268                 commits.append(line.strip())
2269             commits.reverse()
2270
2271         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2272             self.checkAuthorship = False
2273         else:
2274             self.checkAuthorship = True
2275
2276         if self.preserveUser:
2277             self.checkValidP4Users(commits)
2278
2279         #
2280         # Build up a set of options to be passed to diff when
2281         # submitting each commit to p4.
2282         #
2283         if self.detectRenames:
2284             # command-line -M arg
2285             self.diffOpts = "-M"
2286         else:
2287             # If not explicitly set check the config variable
2288             detectRenames = gitConfig("git-p4.detectRenames")
2289
2290             if detectRenames.lower() == "false" or detectRenames == "":
2291                 self.diffOpts = ""
2292             elif detectRenames.lower() == "true":
2293                 self.diffOpts = "-M"
2294             else:
2295                 self.diffOpts = "-M%s" % detectRenames
2296
2297         # no command-line arg for -C or --find-copies-harder, just
2298         # config variables
2299         detectCopies = gitConfig("git-p4.detectCopies")
2300         if detectCopies.lower() == "false" or detectCopies == "":
2301             pass
2302         elif detectCopies.lower() == "true":
2303             self.diffOpts += " -C"
2304         else:
2305             self.diffOpts += " -C%s" % detectCopies
2306
2307         if gitConfigBool("git-p4.detectCopiesHarder"):
2308             self.diffOpts += " --find-copies-harder"
2309
2310         num_shelves = len(self.update_shelve)
2311         if num_shelves > 0 and num_shelves != len(commits):
2312             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2313                      (len(commits), num_shelves))
2314
2315         hooks_path = gitConfig("core.hooksPath")
2316         if len(hooks_path) <= 0:
2317             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2318
2319         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2320         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2321             sys.exit(1)
2322
2323         #
2324         # Apply the commits, one at a time.  On failure, ask if should
2325         # continue to try the rest of the patches, or quit.
2326         #
2327         if self.dry_run:
2328             print("Would apply")
2329         applied = []
2330         last = len(commits) - 1
2331         for i, commit in enumerate(commits):
2332             if self.dry_run:
2333                 print(" ", read_pipe(["git", "show", "-s",
2334                                       "--format=format:%h %s", commit]))
2335                 ok = True
2336             else:
2337                 ok = self.applyCommit(commit)
2338             if ok:
2339                 applied.append(commit)
2340             else:
2341                 if self.prepare_p4_only and i < last:
2342                     print("Processing only the first commit due to option" \
2343                           " --prepare-p4-only")
2344                     break
2345                 if i < last:
2346                     quit = False
2347                     while True:
2348                         # prompt for what to do, or use the option/variable
2349                         if self.conflict_behavior == "ask":
2350                             print("What do you want to do?")
2351                             response = raw_input("[s]kip this commit but apply"
2352                                                  " the rest, or [q]uit? ")
2353                             if not response:
2354                                 continue
2355                         elif self.conflict_behavior == "skip":
2356                             response = "s"
2357                         elif self.conflict_behavior == "quit":
2358                             response = "q"
2359                         else:
2360                             die("Unknown conflict_behavior '%s'" %
2361                                 self.conflict_behavior)
2362
2363                         if response[0] == "s":
2364                             print("Skipping this commit, but applying the rest")
2365                             break
2366                         if response[0] == "q":
2367                             print("Quitting")
2368                             quit = True
2369                             break
2370                     if quit:
2371                         break
2372
2373         chdir(self.oldWorkingDirectory)
2374         shelved_applied = "shelved" if self.shelve else "applied"
2375         if self.dry_run:
2376             pass
2377         elif self.prepare_p4_only:
2378             pass
2379         elif len(commits) == len(applied):
2380             print("All commits {0}!".format(shelved_applied))
2381
2382             sync = P4Sync()
2383             if self.branch:
2384                 sync.branch = self.branch
2385             if self.disable_p4sync:
2386                 sync.sync_origin_only()
2387             else:
2388                 sync.run([])
2389
2390                 if not self.disable_rebase:
2391                     rebase = P4Rebase()
2392                     rebase.rebase()
2393
2394         else:
2395             if len(applied) == 0:
2396                 print("No commits {0}.".format(shelved_applied))
2397             else:
2398                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2399                 for c in commits:
2400                     if c in applied:
2401                         star = "*"
2402                     else:
2403                         star = " "
2404                     print(star, read_pipe(["git", "show", "-s",
2405                                            "--format=format:%h %s",  c]))
2406                 print("You will have to do 'git p4 sync' and rebase.")
2407
2408         if gitConfigBool("git-p4.exportLabels"):
2409             self.exportLabels = True
2410
2411         if self.exportLabels:
2412             p4Labels = getP4Labels(self.depotPath)
2413             gitTags = getGitTags()
2414
2415             missingGitTags = gitTags - p4Labels
2416             self.exportGitTags(missingGitTags)
2417
2418         # exit with error unless everything applied perfectly
2419         if len(commits) != len(applied):
2420                 sys.exit(1)
2421
2422         return True
2423
2424 class View(object):
2425     """Represent a p4 view ("p4 help views"), and map files in a
2426        repo according to the view."""
2427
2428     def __init__(self, client_name):
2429         self.mappings = []
2430         self.client_prefix = "//%s/" % client_name
2431         # cache results of "p4 where" to lookup client file locations
2432         self.client_spec_path_cache = {}
2433
2434     def append(self, view_line):
2435         """Parse a view line, splitting it into depot and client
2436            sides.  Append to self.mappings, preserving order.  This
2437            is only needed for tag creation."""
2438
2439         # Split the view line into exactly two words.  P4 enforces
2440         # structure on these lines that simplifies this quite a bit.
2441         #
2442         # Either or both words may be double-quoted.
2443         # Single quotes do not matter.
2444         # Double-quote marks cannot occur inside the words.
2445         # A + or - prefix is also inside the quotes.
2446         # There are no quotes unless they contain a space.
2447         # The line is already white-space stripped.
2448         # The two words are separated by a single space.
2449         #
2450         if view_line[0] == '"':
2451             # First word is double quoted.  Find its end.
2452             close_quote_index = view_line.find('"', 1)
2453             if close_quote_index <= 0:
2454                 die("No first-word closing quote found: %s" % view_line)
2455             depot_side = view_line[1:close_quote_index]
2456             # skip closing quote and space
2457             rhs_index = close_quote_index + 1 + 1
2458         else:
2459             space_index = view_line.find(" ")
2460             if space_index <= 0:
2461                 die("No word-splitting space found: %s" % view_line)
2462             depot_side = view_line[0:space_index]
2463             rhs_index = space_index + 1
2464
2465         # prefix + means overlay on previous mapping
2466         if depot_side.startswith("+"):
2467             depot_side = depot_side[1:]
2468
2469         # prefix - means exclude this path, leave out of mappings
2470         exclude = False
2471         if depot_side.startswith("-"):
2472             exclude = True
2473             depot_side = depot_side[1:]
2474
2475         if not exclude:
2476             self.mappings.append(depot_side)
2477
2478     def convert_client_path(self, clientFile):
2479         # chop off //client/ part to make it relative
2480         if not clientFile.startswith(self.client_prefix):
2481             die("No prefix '%s' on clientFile '%s'" %
2482                 (self.client_prefix, clientFile))
2483         return clientFile[len(self.client_prefix):]
2484
2485     def update_client_spec_path_cache(self, files):
2486         """ Caching file paths by "p4 where" batch query """
2487
2488         # List depot file paths exclude that already cached
2489         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2490
2491         if len(fileArgs) == 0:
2492             return  # All files in cache
2493
2494         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2495         for res in where_result:
2496             if "code" in res and res["code"] == "error":
2497                 # assume error is "... file(s) not in client view"
2498                 continue
2499             if "clientFile" not in res:
2500                 die("No clientFile in 'p4 where' output")
2501             if "unmap" in res:
2502                 # it will list all of them, but only one not unmap-ped
2503                 continue
2504             if gitConfigBool("core.ignorecase"):
2505                 res['depotFile'] = res['depotFile'].lower()
2506             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2507
2508         # not found files or unmap files set to ""
2509         for depotFile in fileArgs:
2510             if gitConfigBool("core.ignorecase"):
2511                 depotFile = depotFile.lower()
2512             if depotFile not in self.client_spec_path_cache:
2513                 self.client_spec_path_cache[depotFile] = ""
2514
2515     def map_in_client(self, depot_path):
2516         """Return the relative location in the client where this
2517            depot file should live.  Returns "" if the file should
2518            not be mapped in the client."""
2519
2520         if gitConfigBool("core.ignorecase"):
2521             depot_path = depot_path.lower()
2522
2523         if depot_path in self.client_spec_path_cache:
2524             return self.client_spec_path_cache[depot_path]
2525
2526         die( "Error: %s is not found in client spec path" % depot_path )
2527         return ""
2528
2529 class P4Sync(Command, P4UserMap):
2530
2531     def __init__(self):
2532         Command.__init__(self)
2533         P4UserMap.__init__(self)
2534         self.options = [
2535                 optparse.make_option("--branch", dest="branch"),
2536                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2537                 optparse.make_option("--changesfile", dest="changesFile"),
2538                 optparse.make_option("--silent", dest="silent", action="store_true"),
2539                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2540                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2541                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2542                                      help="Import into refs/heads/ , not refs/remotes"),
2543                 optparse.make_option("--max-changes", dest="maxChanges",
2544                                      help="Maximum number of changes to import"),
2545                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2546                                      help="Internal block size to use when iteratively calling p4 changes"),
2547                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2548                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2549                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2550                                      help="Only sync files that are included in the Perforce Client Spec"),
2551                 optparse.make_option("-/", dest="cloneExclude",
2552                                      action="append", type="string",
2553                                      help="exclude depot path"),
2554         ]
2555         self.description = """Imports from Perforce into a git repository.\n
2556     example:
2557     //depot/my/project/ -- to import the current head
2558     //depot/my/project/@all -- to import everything
2559     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2560
2561     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2562
2563         self.usage += " //depot/path[@revRange]"
2564         self.silent = False
2565         self.createdBranches = set()
2566         self.committedChanges = set()
2567         self.branch = ""
2568         self.detectBranches = False
2569         self.detectLabels = False
2570         self.importLabels = False
2571         self.changesFile = ""
2572         self.syncWithOrigin = True
2573         self.importIntoRemotes = True
2574         self.maxChanges = ""
2575         self.changes_block_size = None
2576         self.keepRepoPath = False
2577         self.depotPaths = None
2578         self.p4BranchesInGit = []
2579         self.cloneExclude = []
2580         self.useClientSpec = False
2581         self.useClientSpec_from_options = False
2582         self.clientSpecDirs = None
2583         self.tempBranches = []
2584         self.tempBranchLocation = "refs/git-p4-tmp"
2585         self.largeFileSystem = None
2586         self.suppress_meta_comment = False
2587
2588         if gitConfig('git-p4.largeFileSystem'):
2589             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2590             self.largeFileSystem = largeFileSystemConstructor(
2591                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2592             )
2593
2594         if gitConfig("git-p4.syncFromOrigin") == "false":
2595             self.syncWithOrigin = False
2596
2597         self.depotPaths = []
2598         self.changeRange = ""
2599         self.previousDepotPaths = []
2600         self.hasOrigin = False
2601
2602         # map from branch depot path to parent branch
2603         self.knownBranches = {}
2604         self.initialParents = {}
2605
2606         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2607         self.labels = {}
2608
2609     # Force a checkpoint in fast-import and wait for it to finish
2610     def checkpoint(self):
2611         self.gitStream.write("checkpoint\n\n")
2612         self.gitStream.write("progress checkpoint\n\n")
2613         out = self.gitOutput.readline()
2614         if self.verbose:
2615             print("checkpoint finished: " + out)
2616
2617     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2618         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2619                              for path in self.cloneExclude]
2620         files = []
2621         fnum = 0
2622         while "depotFile%s" % fnum in commit:
2623             path =  commit["depotFile%s" % fnum]
2624
2625             if [p for p in self.cloneExclude
2626                 if p4PathStartsWith(path, p)]:
2627                 found = False
2628             else:
2629                 found = [p for p in self.depotPaths
2630                          if p4PathStartsWith(path, p)]
2631             if not found:
2632                 fnum = fnum + 1
2633                 continue
2634
2635             file = {}
2636             file["path"] = path
2637             file["rev"] = commit["rev%s" % fnum]
2638             file["action"] = commit["action%s" % fnum]
2639             file["type"] = commit["type%s" % fnum]
2640             if shelved:
2641                 file["shelved_cl"] = int(shelved_cl)
2642             files.append(file)
2643             fnum = fnum + 1
2644         return files
2645
2646     def extractJobsFromCommit(self, commit):
2647         jobs = []
2648         jnum = 0
2649         while "job%s" % jnum in commit:
2650             job = commit["job%s" % jnum]
2651             jobs.append(job)
2652             jnum = jnum + 1
2653         return jobs
2654
2655     def stripRepoPath(self, path, prefixes):
2656         """When streaming files, this is called to map a p4 depot path
2657            to where it should go in git.  The prefixes are either
2658            self.depotPaths, or self.branchPrefixes in the case of
2659            branch detection."""
2660
2661         if self.useClientSpec:
2662             # branch detection moves files up a level (the branch name)
2663             # from what client spec interpretation gives
2664             path = self.clientSpecDirs.map_in_client(path)
2665             if self.detectBranches:
2666                 for b in self.knownBranches:
2667                     if path.startswith(b + "/"):
2668                         path = path[len(b)+1:]
2669
2670         elif self.keepRepoPath:
2671             # Preserve everything in relative path name except leading
2672             # //depot/; just look at first prefix as they all should
2673             # be in the same depot.
2674             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2675             if p4PathStartsWith(path, depot):
2676                 path = path[len(depot):]
2677
2678         else:
2679             for p in prefixes:
2680                 if p4PathStartsWith(path, p):
2681                     path = path[len(p):]
2682                     break
2683
2684         path = wildcard_decode(path)
2685         return path
2686
2687     def splitFilesIntoBranches(self, commit):
2688         """Look at each depotFile in the commit to figure out to what
2689            branch it belongs."""
2690
2691         if self.clientSpecDirs:
2692             files = self.extractFilesFromCommit(commit)
2693             self.clientSpecDirs.update_client_spec_path_cache(files)
2694
2695         branches = {}
2696         fnum = 0
2697         while "depotFile%s" % fnum in commit:
2698             path =  commit["depotFile%s" % fnum]
2699             found = [p for p in self.depotPaths
2700                      if p4PathStartsWith(path, p)]
2701             if not found:
2702                 fnum = fnum + 1
2703                 continue
2704
2705             file = {}
2706             file["path"] = path
2707             file["rev"] = commit["rev%s" % fnum]
2708             file["action"] = commit["action%s" % fnum]
2709             file["type"] = commit["type%s" % fnum]
2710             fnum = fnum + 1
2711
2712             # start with the full relative path where this file would
2713             # go in a p4 client
2714             if self.useClientSpec:
2715                 relPath = self.clientSpecDirs.map_in_client(path)
2716             else:
2717                 relPath = self.stripRepoPath(path, self.depotPaths)
2718
2719             for branch in self.knownBranches.keys():
2720                 # add a trailing slash so that a commit into qt/4.2foo
2721                 # doesn't end up in qt/4.2, e.g.
2722                 if relPath.startswith(branch + "/"):
2723                     if branch not in branches:
2724                         branches[branch] = []
2725                     branches[branch].append(file)
2726                     break
2727
2728         return branches
2729
2730     def writeToGitStream(self, gitMode, relPath, contents):
2731         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2732         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2733         for d in contents:
2734             self.gitStream.write(d)
2735         self.gitStream.write('\n')
2736
2737     def encodeWithUTF8(self, path):
2738         try:
2739             path.decode('ascii')
2740         except:
2741             encoding = 'utf8'
2742             if gitConfig('git-p4.pathEncoding'):
2743                 encoding = gitConfig('git-p4.pathEncoding')
2744             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2745             if self.verbose:
2746                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2747         return path
2748
2749     # output one file from the P4 stream
2750     # - helper for streamP4Files
2751
2752     def streamOneP4File(self, file, contents):
2753         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2754         relPath = self.encodeWithUTF8(relPath)
2755         if verbose:
2756             if 'fileSize' in self.stream_file:
2757                 size = int(self.stream_file['fileSize'])
2758             else:
2759                 size = 0 # deleted files don't get a fileSize apparently
2760             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2761             sys.stdout.flush()
2762
2763         (type_base, type_mods) = split_p4_type(file["type"])
2764
2765         git_mode = "100644"
2766         if "x" in type_mods:
2767             git_mode = "100755"
2768         if type_base == "symlink":
2769             git_mode = "120000"
2770             # p4 print on a symlink sometimes contains "target\n";
2771             # if it does, remove the newline
2772             data = ''.join(contents)
2773             if not data:
2774                 # Some version of p4 allowed creating a symlink that pointed
2775                 # to nothing.  This causes p4 errors when checking out such
2776                 # a change, and errors here too.  Work around it by ignoring
2777                 # the bad symlink; hopefully a future change fixes it.
2778                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2779                 return
2780             elif data[-1] == '\n':
2781                 contents = [data[:-1]]
2782             else:
2783                 contents = [data]
2784
2785         if type_base == "utf16":
2786             # p4 delivers different text in the python output to -G
2787             # than it does when using "print -o", or normal p4 client
2788             # operations.  utf16 is converted to ascii or utf8, perhaps.
2789             # But ascii text saved as -t utf16 is completely mangled.
2790             # Invoke print -o to get the real contents.
2791             #
2792             # On windows, the newlines will always be mangled by print, so put
2793             # them back too.  This is not needed to the cygwin windows version,
2794             # just the native "NT" type.
2795             #
2796             try:
2797                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2798             except Exception as e:
2799                 if 'Translation of file content failed' in str(e):
2800                     type_base = 'binary'
2801                 else:
2802                     raise e
2803             else:
2804                 if p4_version_string().find('/NT') >= 0:
2805                     text = text.replace('\r\n', '\n')
2806                 contents = [ text ]
2807
2808         if type_base == "apple":
2809             # Apple filetype files will be streamed as a concatenation of
2810             # its appledouble header and the contents.  This is useless
2811             # on both macs and non-macs.  If using "print -q -o xx", it
2812             # will create "xx" with the data, and "%xx" with the header.
2813             # This is also not very useful.
2814             #
2815             # Ideally, someday, this script can learn how to generate
2816             # appledouble files directly and import those to git, but
2817             # non-mac machines can never find a use for apple filetype.
2818             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2819             return
2820
2821         # Note that we do not try to de-mangle keywords on utf16 files,
2822         # even though in theory somebody may want that.
2823         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2824         if pattern:
2825             regexp = re.compile(pattern, re.VERBOSE)
2826             text = ''.join(contents)
2827             text = regexp.sub(r'$\1$', text)
2828             contents = [ text ]
2829
2830         if self.largeFileSystem:
2831             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2832
2833         self.writeToGitStream(git_mode, relPath, contents)
2834
2835     def streamOneP4Deletion(self, file):
2836         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2837         relPath = self.encodeWithUTF8(relPath)
2838         if verbose:
2839             sys.stdout.write("delete %s\n" % relPath)
2840             sys.stdout.flush()
2841         self.gitStream.write("D %s\n" % relPath)
2842
2843         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2844             self.largeFileSystem.removeLargeFile(relPath)
2845
2846     # handle another chunk of streaming data
2847     def streamP4FilesCb(self, marshalled):
2848
2849         # catch p4 errors and complain
2850         err = None
2851         if "code" in marshalled:
2852             if marshalled["code"] == "error":
2853                 if "data" in marshalled:
2854                     err = marshalled["data"].rstrip()
2855
2856         if not err and 'fileSize' in self.stream_file:
2857             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2858             if required_bytes > 0:
2859                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2860                     os.getcwd(), required_bytes/1024/1024
2861                 )
2862
2863         if err:
2864             f = None
2865             if self.stream_have_file_info:
2866                 if "depotFile" in self.stream_file:
2867                     f = self.stream_file["depotFile"]
2868             # force a failure in fast-import, else an empty
2869             # commit will be made
2870             self.gitStream.write("\n")
2871             self.gitStream.write("die-now\n")
2872             self.gitStream.close()
2873             # ignore errors, but make sure it exits first
2874             self.importProcess.wait()
2875             if f:
2876                 die("Error from p4 print for %s: %s" % (f, err))
2877             else:
2878                 die("Error from p4 print: %s" % err)
2879
2880         if 'depotFile' in marshalled and self.stream_have_file_info:
2881             # start of a new file - output the old one first
2882             self.streamOneP4File(self.stream_file, self.stream_contents)
2883             self.stream_file = {}
2884             self.stream_contents = []
2885             self.stream_have_file_info = False
2886
2887         # pick up the new file information... for the
2888         # 'data' field we need to append to our array
2889         for k in marshalled.keys():
2890             if k == 'data':
2891                 if 'streamContentSize' not in self.stream_file:
2892                     self.stream_file['streamContentSize'] = 0
2893                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2894                 self.stream_contents.append(marshalled['data'])
2895             else:
2896                 self.stream_file[k] = marshalled[k]
2897
2898         if (verbose and
2899             'streamContentSize' in self.stream_file and
2900             'fileSize' in self.stream_file and
2901             'depotFile' in self.stream_file):
2902             size = int(self.stream_file["fileSize"])
2903             if size > 0:
2904                 progress = 100*self.stream_file['streamContentSize']/size
2905                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2906                 sys.stdout.flush()
2907
2908         self.stream_have_file_info = True
2909
2910     # Stream directly from "p4 files" into "git fast-import"
2911     def streamP4Files(self, files):
2912         filesForCommit = []
2913         filesToRead = []
2914         filesToDelete = []
2915
2916         for f in files:
2917             filesForCommit.append(f)
2918             if f['action'] in self.delete_actions:
2919                 filesToDelete.append(f)
2920             else:
2921                 filesToRead.append(f)
2922
2923         # deleted files...
2924         for f in filesToDelete:
2925             self.streamOneP4Deletion(f)
2926
2927         if len(filesToRead) > 0:
2928             self.stream_file = {}
2929             self.stream_contents = []
2930             self.stream_have_file_info = False
2931
2932             # curry self argument
2933             def streamP4FilesCbSelf(entry):
2934                 self.streamP4FilesCb(entry)
2935
2936             fileArgs = []
2937             for f in filesToRead:
2938                 if 'shelved_cl' in f:
2939                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2940                     # the contents
2941                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2942                 else:
2943                     fileArg = '%s#%s' % (f['path'], f['rev'])
2944
2945                 fileArgs.append(fileArg)
2946
2947             p4CmdList(["-x", "-", "print"],
2948                       stdin=fileArgs,
2949                       cb=streamP4FilesCbSelf)
2950
2951             # do the last chunk
2952             if 'depotFile' in self.stream_file:
2953                 self.streamOneP4File(self.stream_file, self.stream_contents)
2954
2955     def make_email(self, userid):
2956         if userid in self.users:
2957             return self.users[userid]
2958         else:
2959             return "%s <a@b>" % userid
2960
2961     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2962         """ Stream a p4 tag.
2963         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2964         """
2965
2966         if verbose:
2967             print("writing tag %s for commit %s" % (labelName, commit))
2968         gitStream.write("tag %s\n" % labelName)
2969         gitStream.write("from %s\n" % commit)
2970
2971         if 'Owner' in labelDetails:
2972             owner = labelDetails["Owner"]
2973         else:
2974             owner = None
2975
2976         # Try to use the owner of the p4 label, or failing that,
2977         # the current p4 user id.
2978         if owner:
2979             email = self.make_email(owner)
2980         else:
2981             email = self.make_email(self.p4UserId())
2982         tagger = "%s %s %s" % (email, epoch, self.tz)
2983
2984         gitStream.write("tagger %s\n" % tagger)
2985
2986         print("labelDetails=",labelDetails)
2987         if 'Description' in labelDetails:
2988             description = labelDetails['Description']
2989         else:
2990             description = 'Label from git p4'
2991
2992         gitStream.write("data %d\n" % len(description))
2993         gitStream.write(description)
2994         gitStream.write("\n")
2995
2996     def inClientSpec(self, path):
2997         if not self.clientSpecDirs:
2998             return True
2999         inClientSpec = self.clientSpecDirs.map_in_client(path)
3000         if not inClientSpec and self.verbose:
3001             print('Ignoring file outside of client spec: {0}'.format(path))
3002         return inClientSpec
3003
3004     def hasBranchPrefix(self, path):
3005         if not self.branchPrefixes:
3006             return True
3007         hasPrefix = [p for p in self.branchPrefixes
3008                         if p4PathStartsWith(path, p)]
3009         if not hasPrefix and self.verbose:
3010             print('Ignoring file outside of prefix: {0}'.format(path))
3011         return hasPrefix
3012
3013     def commit(self, details, files, branch, parent = "", allow_empty=False):
3014         epoch = details["time"]
3015         author = details["user"]
3016         jobs = self.extractJobsFromCommit(details)
3017
3018         if self.verbose:
3019             print('commit into {0}'.format(branch))
3020
3021         if self.clientSpecDirs:
3022             self.clientSpecDirs.update_client_spec_path_cache(files)
3023
3024         files = [f for f in files
3025             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3026
3027         if gitConfigBool('git-p4.keepEmptyCommits'):
3028             allow_empty = True
3029
3030         if not files and not allow_empty:
3031             print('Ignoring revision {0} as it would produce an empty commit.'
3032                 .format(details['change']))
3033             return
3034
3035         self.gitStream.write("commit %s\n" % branch)
3036         self.gitStream.write("mark :%s\n" % details["change"])
3037         self.committedChanges.add(int(details["change"]))
3038         committer = ""
3039         if author not in self.users:
3040             self.getUserMapFromPerforceServer()
3041         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3042
3043         self.gitStream.write("committer %s\n" % committer)
3044
3045         self.gitStream.write("data <<EOT\n")
3046         self.gitStream.write(details["desc"])
3047         if len(jobs) > 0:
3048             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3049
3050         if not self.suppress_meta_comment:
3051             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3052                                 (','.join(self.branchPrefixes), details["change"]))
3053             if len(details['options']) > 0:
3054                 self.gitStream.write(": options = %s" % details['options'])
3055             self.gitStream.write("]\n")
3056
3057         self.gitStream.write("EOT\n\n")
3058
3059         if len(parent) > 0:
3060             if self.verbose:
3061                 print("parent %s" % parent)
3062             self.gitStream.write("from %s\n" % parent)
3063
3064         self.streamP4Files(files)
3065         self.gitStream.write("\n")
3066
3067         change = int(details["change"])
3068
3069         if change in self.labels:
3070             label = self.labels[change]
3071             labelDetails = label[0]
3072             labelRevisions = label[1]
3073             if self.verbose:
3074                 print("Change %s is labelled %s" % (change, labelDetails))
3075
3076             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3077                                                 for p in self.branchPrefixes])
3078
3079             if len(files) == len(labelRevisions):
3080
3081                 cleanedFiles = {}
3082                 for info in files:
3083                     if info["action"] in self.delete_actions:
3084                         continue
3085                     cleanedFiles[info["depotFile"]] = info["rev"]
3086
3087                 if cleanedFiles == labelRevisions:
3088                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3089
3090                 else:
3091                     if not self.silent:
3092                         print("Tag %s does not match with change %s: files do not match."
3093                                % (labelDetails["label"], change))
3094
3095             else:
3096                 if not self.silent:
3097                     print("Tag %s does not match with change %s: file count is different."
3098                            % (labelDetails["label"], change))
3099
3100     # Build a dictionary of changelists and labels, for "detect-labels" option.
3101     def getLabels(self):
3102         self.labels = {}
3103
3104         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3105         if len(l) > 0 and not self.silent:
3106             print("Finding files belonging to labels in %s" % self.depotPaths)
3107
3108         for output in l:
3109             label = output["label"]
3110             revisions = {}
3111             newestChange = 0
3112             if self.verbose:
3113                 print("Querying files for label %s" % label)
3114             for file in p4CmdList(["files"] +
3115                                       ["%s...@%s" % (p, label)
3116                                           for p in self.depotPaths]):
3117                 revisions[file["depotFile"]] = file["rev"]
3118                 change = int(file["change"])
3119                 if change > newestChange:
3120                     newestChange = change
3121
3122             self.labels[newestChange] = [output, revisions]
3123
3124         if self.verbose:
3125             print("Label changes: %s" % self.labels.keys())
3126
3127     # Import p4 labels as git tags. A direct mapping does not
3128     # exist, so assume that if all the files are at the same revision
3129     # then we can use that, or it's something more complicated we should
3130     # just ignore.
3131     def importP4Labels(self, stream, p4Labels):
3132         if verbose:
3133             print("import p4 labels: " + ' '.join(p4Labels))
3134
3135         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3136         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3137         if len(validLabelRegexp) == 0:
3138             validLabelRegexp = defaultLabelRegexp
3139         m = re.compile(validLabelRegexp)
3140
3141         for name in p4Labels:
3142             commitFound = False
3143
3144             if not m.match(name):
3145                 if verbose:
3146                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3147                 continue
3148
3149             if name in ignoredP4Labels:
3150                 continue
3151
3152             labelDetails = p4CmdList(['label', "-o", name])[0]
3153
3154             # get the most recent changelist for each file in this label
3155             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3156                                 for p in self.depotPaths])
3157
3158             if 'change' in change:
3159                 # find the corresponding git commit; take the oldest commit
3160                 changelist = int(change['change'])
3161                 if changelist in self.committedChanges:
3162                     gitCommit = ":%d" % changelist       # use a fast-import mark
3163                     commitFound = True
3164                 else:
3165                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3166                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3167                     if len(gitCommit) == 0:
3168                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3169                     else:
3170                         commitFound = True
3171                         gitCommit = gitCommit.strip()
3172
3173                 if commitFound:
3174                     # Convert from p4 time format
3175                     try:
3176                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3177                     except ValueError:
3178                         print("Could not convert label time %s" % labelDetails['Update'])
3179                         tmwhen = 1
3180
3181                     when = int(time.mktime(tmwhen))
3182                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3183                     if verbose:
3184                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3185             else:
3186                 if verbose:
3187                     print("Label %s has no changelists - possibly deleted?" % name)
3188
3189             if not commitFound:
3190                 # We can't import this label; don't try again as it will get very
3191                 # expensive repeatedly fetching all the files for labels that will
3192                 # never be imported. If the label is moved in the future, the
3193                 # ignore will need to be removed manually.
3194                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3195
3196     def guessProjectName(self):
3197         for p in self.depotPaths:
3198             if p.endswith("/"):
3199                 p = p[:-1]
3200             p = p[p.strip().rfind("/") + 1:]
3201             if not p.endswith("/"):
3202                p += "/"
3203             return p
3204
3205     def getBranchMapping(self):
3206         lostAndFoundBranches = set()
3207
3208         user = gitConfig("git-p4.branchUser")
3209         if len(user) > 0:
3210             command = "branches -u %s" % user
3211         else:
3212             command = "branches"
3213
3214         for info in p4CmdList(command):
3215             details = p4Cmd(["branch", "-o", info["branch"]])
3216             viewIdx = 0
3217             while "View%s" % viewIdx in details:
3218                 paths = details["View%s" % viewIdx].split(" ")
3219                 viewIdx = viewIdx + 1
3220                 # require standard //depot/foo/... //depot/bar/... mapping
3221                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3222                     continue
3223                 source = paths[0]
3224                 destination = paths[1]
3225                 ## HACK
3226                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3227                     source = source[len(self.depotPaths[0]):-4]
3228                     destination = destination[len(self.depotPaths[0]):-4]
3229
3230                     if destination in self.knownBranches:
3231                         if not self.silent:
3232                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3233                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3234                         continue
3235
3236                     self.knownBranches[destination] = source
3237
3238                     lostAndFoundBranches.discard(destination)
3239
3240                     if source not in self.knownBranches:
3241                         lostAndFoundBranches.add(source)
3242
3243         # Perforce does not strictly require branches to be defined, so we also
3244         # check git config for a branch list.
3245         #
3246         # Example of branch definition in git config file:
3247         # [git-p4]
3248         #   branchList=main:branchA
3249         #   branchList=main:branchB
3250         #   branchList=branchA:branchC
3251         configBranches = gitConfigList("git-p4.branchList")
3252         for branch in configBranches:
3253             if branch:
3254                 (source, destination) = branch.split(":")
3255                 self.knownBranches[destination] = source
3256
3257                 lostAndFoundBranches.discard(destination)
3258
3259                 if source not in self.knownBranches:
3260                     lostAndFoundBranches.add(source)
3261
3262
3263         for branch in lostAndFoundBranches:
3264             self.knownBranches[branch] = branch
3265
3266     def getBranchMappingFromGitBranches(self):
3267         branches = p4BranchesInGit(self.importIntoRemotes)
3268         for branch in branches.keys():
3269             if branch == "master":
3270                 branch = "main"
3271             else:
3272                 branch = branch[len(self.projectName):]
3273             self.knownBranches[branch] = branch
3274
3275     def updateOptionDict(self, d):
3276         option_keys = {}
3277         if self.keepRepoPath:
3278             option_keys['keepRepoPath'] = 1
3279
3280         d["options"] = ' '.join(sorted(option_keys.keys()))
3281
3282     def readOptions(self, d):
3283         self.keepRepoPath = ('options' in d
3284                              and ('keepRepoPath' in d['options']))
3285
3286     def gitRefForBranch(self, branch):
3287         if branch == "main":
3288             return self.refPrefix + "master"
3289
3290         if len(branch) <= 0:
3291             return branch
3292
3293         return self.refPrefix + self.projectName + branch
3294
3295     def gitCommitByP4Change(self, ref, change):
3296         if self.verbose:
3297             print("looking in ref " + ref + " for change %s using bisect..." % change)
3298
3299         earliestCommit = ""
3300         latestCommit = parseRevision(ref)
3301
3302         while True:
3303             if self.verbose:
3304                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3305             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3306             if len(next) == 0:
3307                 if self.verbose:
3308                     print("argh")
3309                 return ""
3310             log = extractLogMessageFromGitCommit(next)
3311             settings = extractSettingsGitLog(log)
3312             currentChange = int(settings['change'])
3313             if self.verbose:
3314                 print("current change %s" % currentChange)
3315
3316             if currentChange == change:
3317                 if self.verbose:
3318                     print("found %s" % next)
3319                 return next
3320
3321             if currentChange < change:
3322                 earliestCommit = "^%s" % next
3323             else:
3324                 latestCommit = "%s" % next
3325
3326         return ""
3327
3328     def importNewBranch(self, branch, maxChange):
3329         # make fast-import flush all changes to disk and update the refs using the checkpoint
3330         # command so that we can try to find the branch parent in the git history
3331         self.gitStream.write("checkpoint\n\n");
3332         self.gitStream.flush();
3333         branchPrefix = self.depotPaths[0] + branch + "/"
3334         range = "@1,%s" % maxChange
3335         #print "prefix" + branchPrefix
3336         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3337         if len(changes) <= 0:
3338             return False
3339         firstChange = changes[0]
3340         #print "first change in branch: %s" % firstChange
3341         sourceBranch = self.knownBranches[branch]
3342         sourceDepotPath = self.depotPaths[0] + sourceBranch
3343         sourceRef = self.gitRefForBranch(sourceBranch)
3344         #print "source " + sourceBranch
3345
3346         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3347         #print "branch parent: %s" % branchParentChange
3348         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3349         if len(gitParent) > 0:
3350             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3351             #print "parent git commit: %s" % gitParent
3352
3353         self.importChanges(changes)
3354         return True
3355
3356     def searchParent(self, parent, branch, target):
3357         parentFound = False
3358         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3359                                      "--no-merges", parent]):
3360             blob = blob.strip()
3361             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3362                 parentFound = True
3363                 if self.verbose:
3364                     print("Found parent of %s in commit %s" % (branch, blob))
3365                 break
3366         if parentFound:
3367             return blob
3368         else:
3369             return None
3370
3371     def importChanges(self, changes, origin_revision=0):
3372         cnt = 1
3373         for change in changes:
3374             description = p4_describe(change)
3375             self.updateOptionDict(description)
3376
3377             if not self.silent:
3378                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3379                 sys.stdout.flush()
3380             cnt = cnt + 1
3381
3382             try:
3383                 if self.detectBranches:
3384                     branches = self.splitFilesIntoBranches(description)
3385                     for branch in branches.keys():
3386                         ## HACK  --hwn
3387                         branchPrefix = self.depotPaths[0] + branch + "/"
3388                         self.branchPrefixes = [ branchPrefix ]
3389
3390                         parent = ""
3391
3392                         filesForCommit = branches[branch]
3393
3394                         if self.verbose:
3395                             print("branch is %s" % branch)
3396
3397                         self.updatedBranches.add(branch)
3398
3399                         if branch not in self.createdBranches:
3400                             self.createdBranches.add(branch)
3401                             parent = self.knownBranches[branch]
3402                             if parent == branch:
3403                                 parent = ""
3404                             else:
3405                                 fullBranch = self.projectName + branch
3406                                 if fullBranch not in self.p4BranchesInGit:
3407                                     if not self.silent:
3408                                         print("\n    Importing new branch %s" % fullBranch);
3409                                     if self.importNewBranch(branch, change - 1):
3410                                         parent = ""
3411                                         self.p4BranchesInGit.append(fullBranch)
3412                                     if not self.silent:
3413                                         print("\n    Resuming with change %s" % change);
3414
3415                                 if self.verbose:
3416                                     print("parent determined through known branches: %s" % parent)
3417
3418                         branch = self.gitRefForBranch(branch)
3419                         parent = self.gitRefForBranch(parent)
3420
3421                         if self.verbose:
3422                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3423
3424                         if len(parent) == 0 and branch in self.initialParents:
3425                             parent = self.initialParents[branch]
3426                             del self.initialParents[branch]
3427
3428                         blob = None
3429                         if len(parent) > 0:
3430                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3431                             if self.verbose:
3432                                 print("Creating temporary branch: " + tempBranch)
3433                             self.commit(description, filesForCommit, tempBranch)
3434                             self.tempBranches.append(tempBranch)
3435                             self.checkpoint()
3436                             blob = self.searchParent(parent, branch, tempBranch)
3437                         if blob:
3438                             self.commit(description, filesForCommit, branch, blob)
3439                         else:
3440                             if self.verbose:
3441                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3442                             self.commit(description, filesForCommit, branch, parent)
3443                 else:
3444                     files = self.extractFilesFromCommit(description)
3445                     self.commit(description, files, self.branch,
3446                                 self.initialParent)
3447                     # only needed once, to connect to the previous commit
3448                     self.initialParent = ""
3449             except IOError:
3450                 print(self.gitError.read())
3451                 sys.exit(1)
3452
3453     def sync_origin_only(self):
3454         if self.syncWithOrigin:
3455             self.hasOrigin = originP4BranchesExist()
3456             if self.hasOrigin:
3457                 if not self.silent:
3458                     print('Syncing with origin first, using "git fetch origin"')
3459                 system("git fetch origin")
3460
3461     def importHeadRevision(self, revision):
3462         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3463
3464         details = {}
3465         details["user"] = "git perforce import user"
3466         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3467                            % (' '.join(self.depotPaths), revision))
3468         details["change"] = revision
3469         newestRevision = 0
3470
3471         fileCnt = 0
3472         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3473
3474         for info in p4CmdList(["files"] + fileArgs):
3475
3476             if 'code' in info and info['code'] == 'error':
3477                 sys.stderr.write("p4 returned an error: %s\n"
3478                                  % info['data'])
3479                 if info['data'].find("must refer to client") >= 0:
3480                     sys.stderr.write("This particular p4 error is misleading.\n")
3481                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3482                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3483                 sys.exit(1)
3484             if 'p4ExitCode' in info:
3485                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3486                 sys.exit(1)
3487
3488
3489             change = int(info["change"])
3490             if change > newestRevision:
3491                 newestRevision = change
3492
3493             if info["action"] in self.delete_actions:
3494                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3495                 #fileCnt = fileCnt + 1
3496                 continue
3497
3498             for prop in ["depotFile", "rev", "action", "type" ]:
3499                 details["%s%s" % (prop, fileCnt)] = info[prop]
3500
3501             fileCnt = fileCnt + 1
3502
3503         details["change"] = newestRevision
3504
3505         # Use time from top-most change so that all git p4 clones of
3506         # the same p4 repo have the same commit SHA1s.
3507         res = p4_describe(newestRevision)
3508         details["time"] = res["time"]
3509
3510         self.updateOptionDict(details)
3511         try:
3512             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3513         except IOError:
3514             print("IO error with git fast-import. Is your git version recent enough?")
3515             print(self.gitError.read())
3516
3517     def openStreams(self):
3518         self.importProcess = subprocess.Popen(["git", "fast-import"],
3519                                               stdin=subprocess.PIPE,
3520                                               stdout=subprocess.PIPE,
3521                                               stderr=subprocess.PIPE);
3522         self.gitOutput = self.importProcess.stdout
3523         self.gitStream = self.importProcess.stdin
3524         self.gitError = self.importProcess.stderr
3525
3526     def closeStreams(self):
3527         self.gitStream.close()
3528         if self.importProcess.wait() != 0:
3529             die("fast-import failed: %s" % self.gitError.read())
3530         self.gitOutput.close()
3531         self.gitError.close()
3532
3533     def run(self, args):
3534         if self.importIntoRemotes:
3535             self.refPrefix = "refs/remotes/p4/"
3536         else:
3537             self.refPrefix = "refs/heads/p4/"
3538
3539         self.sync_origin_only()
3540
3541         branch_arg_given = bool(self.branch)
3542         if len(self.branch) == 0:
3543             self.branch = self.refPrefix + "master"
3544             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3545                 system("git update-ref %s refs/heads/p4" % self.branch)
3546                 system("git branch -D p4")
3547
3548         # accept either the command-line option, or the configuration variable
3549         if self.useClientSpec:
3550             # will use this after clone to set the variable
3551             self.useClientSpec_from_options = True
3552         else:
3553             if gitConfigBool("git-p4.useclientspec"):
3554                 self.useClientSpec = True
3555         if self.useClientSpec:
3556             self.clientSpecDirs = getClientSpec()
3557
3558         # TODO: should always look at previous commits,
3559         # merge with previous imports, if possible.
3560         if args == []:
3561             if self.hasOrigin:
3562                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3563
3564             # branches holds mapping from branch name to sha1
3565             branches = p4BranchesInGit(self.importIntoRemotes)
3566
3567             # restrict to just this one, disabling detect-branches
3568             if branch_arg_given:
3569                 short = self.branch.split("/")[-1]
3570                 if short in branches:
3571                     self.p4BranchesInGit = [ short ]
3572             else:
3573                 self.p4BranchesInGit = branches.keys()
3574
3575             if len(self.p4BranchesInGit) > 1:
3576                 if not self.silent:
3577                     print("Importing from/into multiple branches")
3578                 self.detectBranches = True
3579                 for branch in branches.keys():
3580                     self.initialParents[self.refPrefix + branch] = \
3581                         branches[branch]
3582
3583             if self.verbose:
3584                 print("branches: %s" % self.p4BranchesInGit)
3585
3586             p4Change = 0
3587             for branch in self.p4BranchesInGit:
3588                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3589
3590                 settings = extractSettingsGitLog(logMsg)
3591
3592                 self.readOptions(settings)
3593                 if ('depot-paths' in settings
3594                     and 'change' in settings):
3595                     change = int(settings['change']) + 1
3596                     p4Change = max(p4Change, change)
3597
3598                     depotPaths = sorted(settings['depot-paths'])
3599                     if self.previousDepotPaths == []:
3600                         self.previousDepotPaths = depotPaths
3601                     else:
3602                         paths = []
3603                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3604                             prev_list = prev.split("/")
3605                             cur_list = cur.split("/")
3606                             for i in range(0, min(len(cur_list), len(prev_list))):
3607                                 if cur_list[i] != prev_list[i]:
3608                                     i = i - 1
3609                                     break
3610
3611                             paths.append ("/".join(cur_list[:i + 1]))
3612
3613                         self.previousDepotPaths = paths
3614
3615             if p4Change > 0:
3616                 self.depotPaths = sorted(self.previousDepotPaths)
3617                 self.changeRange = "@%s,#head" % p4Change
3618                 if not self.silent and not self.detectBranches:
3619                     print("Performing incremental import into %s git branch" % self.branch)
3620
3621         # accept multiple ref name abbreviations:
3622         #    refs/foo/bar/branch -> use it exactly
3623         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3624         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3625         if not self.branch.startswith("refs/"):
3626             if self.importIntoRemotes:
3627                 prepend = "refs/remotes/"
3628             else:
3629                 prepend = "refs/heads/"
3630             if not self.branch.startswith("p4/"):
3631                 prepend += "p4/"
3632             self.branch = prepend + self.branch
3633
3634         if len(args) == 0 and self.depotPaths:
3635             if not self.silent:
3636                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3637         else:
3638             if self.depotPaths and self.depotPaths != args:
3639                 print("previous import used depot path %s and now %s was specified. "
3640                        "This doesn't work!" % (' '.join (self.depotPaths),
3641                                                ' '.join (args)))
3642                 sys.exit(1)
3643
3644             self.depotPaths = sorted(args)
3645
3646         revision = ""
3647         self.users = {}
3648
3649         # Make sure no revision specifiers are used when --changesfile
3650         # is specified.
3651         bad_changesfile = False
3652         if len(self.changesFile) > 0:
3653             for p in self.depotPaths:
3654                 if p.find("@") >= 0 or p.find("#") >= 0:
3655                     bad_changesfile = True
3656                     break
3657         if bad_changesfile:
3658             die("Option --changesfile is incompatible with revision specifiers")
3659
3660         newPaths = []
3661         for p in self.depotPaths:
3662             if p.find("@") != -1:
3663                 atIdx = p.index("@")
3664                 self.changeRange = p[atIdx:]
3665                 if self.changeRange == "@all":
3666                     self.changeRange = ""
3667                 elif ',' not in self.changeRange:
3668                     revision = self.changeRange
3669                     self.changeRange = ""
3670                 p = p[:atIdx]
3671             elif p.find("#") != -1:
3672                 hashIdx = p.index("#")
3673                 revision = p[hashIdx:]
3674                 p = p[:hashIdx]
3675             elif self.previousDepotPaths == []:
3676                 # pay attention to changesfile, if given, else import
3677                 # the entire p4 tree at the head revision
3678                 if len(self.changesFile) == 0:
3679                     revision = "#head"
3680
3681             p = re.sub ("\.\.\.$", "", p)
3682             if not p.endswith("/"):
3683                 p += "/"
3684
3685             newPaths.append(p)
3686
3687         self.depotPaths = newPaths
3688
3689         # --detect-branches may change this for each branch
3690         self.branchPrefixes = self.depotPaths
3691
3692         self.loadUserMapFromCache()
3693         self.labels = {}
3694         if self.detectLabels:
3695             self.getLabels();
3696
3697         if self.detectBranches:
3698             ## FIXME - what's a P4 projectName ?
3699             self.projectName = self.guessProjectName()
3700
3701             if self.hasOrigin:
3702                 self.getBranchMappingFromGitBranches()
3703             else:
3704                 self.getBranchMapping()
3705             if self.verbose:
3706                 print("p4-git branches: %s" % self.p4BranchesInGit)
3707                 print("initial parents: %s" % self.initialParents)
3708             for b in self.p4BranchesInGit:
3709                 if b != "master":
3710
3711                     ## FIXME
3712                     b = b[len(self.projectName):]
3713                 self.createdBranches.add(b)
3714
3715         self.openStreams()
3716
3717         if revision:
3718             self.importHeadRevision(revision)
3719         else:
3720             changes = []
3721
3722             if len(self.changesFile) > 0:
3723                 output = open(self.changesFile).readlines()
3724                 changeSet = set()
3725                 for line in output:
3726                     changeSet.add(int(line))
3727
3728                 for change in changeSet:
3729                     changes.append(change)
3730
3731                 changes.sort()
3732             else:
3733                 # catch "git p4 sync" with no new branches, in a repo that
3734                 # does not have any existing p4 branches
3735                 if len(args) == 0:
3736                     if not self.p4BranchesInGit:
3737                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3738
3739                     # The default branch is master, unless --branch is used to
3740                     # specify something else.  Make sure it exists, or complain
3741                     # nicely about how to use --branch.
3742                     if not self.detectBranches:
3743                         if not branch_exists(self.branch):
3744                             if branch_arg_given:
3745                                 die("Error: branch %s does not exist." % self.branch)
3746                             else:
3747                                 die("Error: no branch %s; perhaps specify one with --branch." %
3748                                     self.branch)
3749
3750                 if self.verbose:
3751                     print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3752                                                               self.changeRange))
3753                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3754
3755                 if len(self.maxChanges) > 0:
3756                     changes = changes[:min(int(self.maxChanges), len(changes))]
3757
3758             if len(changes) == 0:
3759                 if not self.silent:
3760                     print("No changes to import!")
3761             else:
3762                 if not self.silent and not self.detectBranches:
3763                     print("Import destination: %s" % self.branch)
3764
3765                 self.updatedBranches = set()
3766
3767                 if not self.detectBranches:
3768                     if args:
3769                         # start a new branch
3770                         self.initialParent = ""
3771                     else:
3772                         # build on a previous revision
3773                         self.initialParent = parseRevision(self.branch)
3774
3775                 self.importChanges(changes)
3776
3777                 if not self.silent:
3778                     print("")
3779                     if len(self.updatedBranches) > 0:
3780                         sys.stdout.write("Updated branches: ")
3781                         for b in self.updatedBranches:
3782                             sys.stdout.write("%s " % b)
3783                         sys.stdout.write("\n")
3784
3785         if gitConfigBool("git-p4.importLabels"):
3786             self.importLabels = True
3787
3788         if self.importLabels:
3789             p4Labels = getP4Labels(self.depotPaths)
3790             gitTags = getGitTags()
3791
3792             missingP4Labels = p4Labels - gitTags
3793             self.importP4Labels(self.gitStream, missingP4Labels)
3794
3795         self.closeStreams()
3796
3797         # Cleanup temporary branches created during import
3798         if self.tempBranches != []:
3799             for branch in self.tempBranches:
3800                 read_pipe("git update-ref -d %s" % branch)
3801             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3802
3803         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3804         # a convenient shortcut refname "p4".
3805         if self.importIntoRemotes:
3806             head_ref = self.refPrefix + "HEAD"
3807             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3808                 system(["git", "symbolic-ref", head_ref, self.branch])
3809
3810         return True
3811
3812 class P4Rebase(Command):
3813     def __init__(self):
3814         Command.__init__(self)
3815         self.options = [
3816                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3817         ]
3818         self.importLabels = False
3819         self.description = ("Fetches the latest revision from perforce and "
3820                             + "rebases the current work (branch) against it")
3821
3822     def run(self, args):
3823         sync = P4Sync()
3824         sync.importLabels = self.importLabels
3825         sync.run([])
3826
3827         return self.rebase()
3828
3829     def rebase(self):
3830         if os.system("git update-index --refresh") != 0:
3831             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3832         if len(read_pipe("git diff-index HEAD --")) > 0:
3833             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3834
3835         [upstream, settings] = findUpstreamBranchPoint()
3836         if len(upstream) == 0:
3837             die("Cannot find upstream branchpoint for rebase")
3838
3839         # the branchpoint may be p4/foo~3, so strip off the parent
3840         upstream = re.sub("~[0-9]+$", "", upstream)
3841
3842         print("Rebasing the current branch onto %s" % upstream)
3843         oldHead = read_pipe("git rev-parse HEAD").strip()
3844         system("git rebase %s" % upstream)
3845         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3846         return True
3847
3848 class P4Clone(P4Sync):
3849     def __init__(self):
3850         P4Sync.__init__(self)
3851         self.description = "Creates a new git repository and imports from Perforce into it"
3852         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3853         self.options += [
3854             optparse.make_option("--destination", dest="cloneDestination",
3855                                  action='store', default=None,
3856                                  help="where to leave result of the clone"),
3857             optparse.make_option("--bare", dest="cloneBare",
3858                                  action="store_true", default=False),
3859         ]
3860         self.cloneDestination = None
3861         self.needsGit = False
3862         self.cloneBare = False
3863
3864     def defaultDestination(self, args):
3865         ## TODO: use common prefix of args?
3866         depotPath = args[0]
3867         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3868         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3869         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3870         depotDir = re.sub(r"/$", "", depotDir)
3871         return os.path.split(depotDir)[1]
3872
3873     def run(self, args):
3874         if len(args) < 1:
3875             return False
3876
3877         if self.keepRepoPath and not self.cloneDestination:
3878             sys.stderr.write("Must specify destination for --keep-path\n")
3879             sys.exit(1)
3880
3881         depotPaths = args
3882
3883         if not self.cloneDestination and len(depotPaths) > 1:
3884             self.cloneDestination = depotPaths[-1]
3885             depotPaths = depotPaths[:-1]
3886
3887         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3888         for p in depotPaths:
3889             if not p.startswith("//"):
3890                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3891                 return False
3892
3893         if not self.cloneDestination:
3894             self.cloneDestination = self.defaultDestination(args)
3895
3896         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3897
3898         if not os.path.exists(self.cloneDestination):
3899             os.makedirs(self.cloneDestination)
3900         chdir(self.cloneDestination)
3901
3902         init_cmd = [ "git", "init" ]
3903         if self.cloneBare:
3904             init_cmd.append("--bare")
3905         retcode = subprocess.call(init_cmd)
3906         if retcode:
3907             raise CalledProcessError(retcode, init_cmd)
3908
3909         if not P4Sync.run(self, depotPaths):
3910             return False
3911
3912         # create a master branch and check out a work tree
3913         if gitBranchExists(self.branch):
3914             system([ "git", "branch", "master", self.branch ])
3915             if not self.cloneBare:
3916                 system([ "git", "checkout", "-f" ])
3917         else:
3918             print('Not checking out any branch, use ' \
3919                   '"git checkout -q -b master <branch>"')
3920
3921         # auto-set this variable if invoked with --use-client-spec
3922         if self.useClientSpec_from_options:
3923             system("git config --bool git-p4.useclientspec true")
3924
3925         return True
3926
3927 class P4Unshelve(Command):
3928     def __init__(self):
3929         Command.__init__(self)
3930         self.options = []
3931         self.origin = "HEAD"
3932         self.description = "Unshelve a P4 changelist into a git commit"
3933         self.usage = "usage: %prog [options] changelist"
3934         self.options += [
3935                 optparse.make_option("--origin", dest="origin",
3936                     help="Use this base revision instead of the default (%s)" % self.origin),
3937         ]
3938         self.verbose = False
3939         self.noCommit = False
3940         self.destbranch = "refs/remotes/p4-unshelved"
3941
3942     def renameBranch(self, branch_name):
3943         """ Rename the existing branch to branch_name.N
3944         """
3945
3946         found = True
3947         for i in range(0,1000):
3948             backup_branch_name = "{0}.{1}".format(branch_name, i)
3949             if not gitBranchExists(backup_branch_name):
3950                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3951                 gitDeleteRef(branch_name)
3952                 found = True
3953                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3954                 break
3955
3956         if not found:
3957             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3958
3959     def findLastP4Revision(self, starting_point):
3960         """ Look back from starting_point for the first commit created by git-p4
3961             to find the P4 commit we are based on, and the depot-paths.
3962         """
3963
3964         for parent in (range(65535)):
3965             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3966             settings = extractSettingsGitLog(log)
3967             if 'change' in settings:
3968                 return settings
3969
3970         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3971
3972     def createShelveParent(self, change, branch_name, sync, origin):
3973         """ Create a commit matching the parent of the shelved changelist 'change'
3974         """
3975         parent_description = p4_describe(change, shelved=True)
3976         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
3977         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
3978
3979         parent_files = []
3980         for f in files:
3981             # if it was added in the shelved changelist, it won't exist in the parent
3982             if f['action'] in self.add_actions:
3983                 continue
3984
3985             # if it was deleted in the shelved changelist it must not be deleted
3986             # in the parent - we might even need to create it if the origin branch
3987             # does not have it
3988             if f['action'] in self.delete_actions:
3989                 f['action'] = 'add'
3990
3991             parent_files.append(f)
3992
3993         sync.commit(parent_description, parent_files, branch_name,
3994                 parent=origin, allow_empty=True)
3995         print("created parent commit for {0} based on {1} in {2}".format(
3996             change, self.origin, branch_name))
3997
3998     def run(self, args):
3999         if len(args) != 1:
4000             return False
4001
4002         if not gitBranchExists(self.origin):
4003             sys.exit("origin branch {0} does not exist".format(self.origin))
4004
4005         sync = P4Sync()
4006         changes = args
4007
4008         # only one change at a time
4009         change = changes[0]
4010
4011         # if the target branch already exists, rename it
4012         branch_name = "{0}/{1}".format(self.destbranch, change)
4013         if gitBranchExists(branch_name):
4014             self.renameBranch(branch_name)
4015         sync.branch = branch_name
4016
4017         sync.verbose = self.verbose
4018         sync.suppress_meta_comment = True
4019
4020         settings = self.findLastP4Revision(self.origin)
4021         sync.depotPaths = settings['depot-paths']
4022         sync.branchPrefixes = sync.depotPaths
4023
4024         sync.openStreams()
4025         sync.loadUserMapFromCache()
4026         sync.silent = True
4027
4028         # create a commit for the parent of the shelved changelist
4029         self.createShelveParent(change, branch_name, sync, self.origin)
4030
4031         # create the commit for the shelved changelist itself
4032         description = p4_describe(change, True)
4033         files = sync.extractFilesFromCommit(description, True, change)
4034
4035         sync.commit(description, files, branch_name, "")
4036         sync.closeStreams()
4037
4038         print("unshelved changelist {0} into {1}".format(change, branch_name))
4039
4040         return True
4041
4042 class P4Branches(Command):
4043     def __init__(self):
4044         Command.__init__(self)
4045         self.options = [ ]
4046         self.description = ("Shows the git branches that hold imports and their "
4047                             + "corresponding perforce depot paths")
4048         self.verbose = False
4049
4050     def run(self, args):
4051         if originP4BranchesExist():
4052             createOrUpdateBranchesFromOrigin()
4053
4054         cmdline = "git rev-parse --symbolic "
4055         cmdline += " --remotes"
4056
4057         for line in read_pipe_lines(cmdline):
4058             line = line.strip()
4059
4060             if not line.startswith('p4/') or line == "p4/HEAD":
4061                 continue
4062             branch = line
4063
4064             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4065             settings = extractSettingsGitLog(log)
4066
4067             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4068         return True
4069
4070 class HelpFormatter(optparse.IndentedHelpFormatter):
4071     def __init__(self):
4072         optparse.IndentedHelpFormatter.__init__(self)
4073
4074     def format_description(self, description):
4075         if description:
4076             return description + "\n"
4077         else:
4078             return ""
4079
4080 def printUsage(commands):
4081     print("usage: %s <command> [options]" % sys.argv[0])
4082     print("")
4083     print("valid commands: %s" % ", ".join(commands))
4084     print("")
4085     print("Try %s <command> --help for command specific help." % sys.argv[0])
4086     print("")
4087
4088 commands = {
4089     "debug" : P4Debug,
4090     "submit" : P4Submit,
4091     "commit" : P4Submit,
4092     "sync" : P4Sync,
4093     "rebase" : P4Rebase,
4094     "clone" : P4Clone,
4095     "rollback" : P4RollBack,
4096     "branches" : P4Branches,
4097     "unshelve" : P4Unshelve,
4098 }
4099
4100
4101 def main():
4102     if len(sys.argv[1:]) == 0:
4103         printUsage(commands.keys())
4104         sys.exit(2)
4105
4106     cmdName = sys.argv[1]
4107     try:
4108         klass = commands[cmdName]
4109         cmd = klass()
4110     except KeyError:
4111         print("unknown command %s" % cmdName)
4112         print("")
4113         printUsage(commands.keys())
4114         sys.exit(2)
4115
4116     options = cmd.options
4117     cmd.gitdir = os.environ.get("GIT_DIR", None)
4118
4119     args = sys.argv[2:]
4120
4121     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4122     if cmd.needsGit:
4123         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4124
4125     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4126                                    options,
4127                                    description = cmd.description,
4128                                    formatter = HelpFormatter())
4129
4130     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4131     global verbose
4132     verbose = cmd.verbose
4133     if cmd.needsGit:
4134         if cmd.gitdir == None:
4135             cmd.gitdir = os.path.abspath(".git")
4136             if not isValidGitDir(cmd.gitdir):
4137                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4138                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4139                 if os.path.exists(cmd.gitdir):
4140                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4141                     if len(cdup) > 0:
4142                         chdir(cdup);
4143
4144         if not isValidGitDir(cmd.gitdir):
4145             if isValidGitDir(cmd.gitdir + "/.git"):
4146                 cmd.gitdir += "/.git"
4147             else:
4148                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4149
4150         # so git commands invoked from the P4 workspace will succeed
4151         os.environ["GIT_DIR"] = cmd.gitdir
4152
4153     if not cmd.run(args):
4154         parser.print_help()
4155         sys.exit(2)
4156
4157
4158 if __name__ == '__main__':
4159     main()