Merge branch 'hw/advise-ng'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
17 #
18 import sys
19 if sys.hexversion < 0x02040000:
20     # The limiter is the subprocess module
21     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
22     sys.exit(1)
23 import os
24 import optparse
25 import marshal
26 import subprocess
27 import tempfile
28 import time
29 import platform
30 import re
31 import shutil
32 import stat
33 import zipfile
34 import zlib
35 import ctypes
36 import errno
37
38 # support basestring in python3
39 try:
40     unicode = unicode
41 except NameError:
42     # 'unicode' is undefined, must be Python 3
43     str = str
44     unicode = str
45     bytes = bytes
46     basestring = (str,bytes)
47 else:
48     # 'unicode' exists, must be Python 2
49     str = str
50     unicode = unicode
51     bytes = str
52     basestring = basestring
53
54 try:
55     from subprocess import CalledProcessError
56 except ImportError:
57     # from python2.7:subprocess.py
58     # Exception classes used by this module.
59     class CalledProcessError(Exception):
60         """This exception is raised when a process run by check_call() returns
61         a non-zero exit status.  The exit status will be stored in the
62         returncode attribute."""
63         def __init__(self, returncode, cmd):
64             self.returncode = returncode
65             self.cmd = cmd
66         def __str__(self):
67             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
68
69 verbose = False
70
71 # Only labels/tags matching this will be imported/exported
72 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
73
74 # The block size is reduced automatically if required
75 defaultBlockSize = 1<<20
76
77 p4_access_checked = False
78
79 def p4_build_cmd(cmd):
80     """Build a suitable p4 command line.
81
82     This consolidates building and returning a p4 command line into one
83     location. It means that hooking into the environment, or other configuration
84     can be done more easily.
85     """
86     real_cmd = ["p4"]
87
88     user = gitConfig("git-p4.user")
89     if len(user) > 0:
90         real_cmd += ["-u",user]
91
92     password = gitConfig("git-p4.password")
93     if len(password) > 0:
94         real_cmd += ["-P", password]
95
96     port = gitConfig("git-p4.port")
97     if len(port) > 0:
98         real_cmd += ["-p", port]
99
100     host = gitConfig("git-p4.host")
101     if len(host) > 0:
102         real_cmd += ["-H", host]
103
104     client = gitConfig("git-p4.client")
105     if len(client) > 0:
106         real_cmd += ["-c", client]
107
108     retries = gitConfigInt("git-p4.retries")
109     if retries is None:
110         # Perform 3 retries by default
111         retries = 3
112     if retries > 0:
113         # Provide a way to not pass this option by setting git-p4.retries to 0
114         real_cmd += ["-r", str(retries)]
115
116     if isinstance(cmd,basestring):
117         real_cmd = ' '.join(real_cmd) + ' ' + cmd
118     else:
119         real_cmd += cmd
120
121     # now check that we can actually talk to the server
122     global p4_access_checked
123     if not p4_access_checked:
124         p4_access_checked = True    # suppress access checks in p4_check_access itself
125         p4_check_access()
126
127     return real_cmd
128
129 def git_dir(path):
130     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
131         This won't automatically add ".git" to a directory.
132     """
133     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
134     if not d or len(d) == 0:
135         return None
136     else:
137         return d
138
139 def chdir(path, is_client_path=False):
140     """Do chdir to the given path, and set the PWD environment
141        variable for use by P4.  It does not look at getcwd() output.
142        Since we're not using the shell, it is necessary to set the
143        PWD environment variable explicitly.
144
145        Normally, expand the path to force it to be absolute.  This
146        addresses the use of relative path names inside P4 settings,
147        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
148        as given; it looks for .p4config using PWD.
149
150        If is_client_path, the path was handed to us directly by p4,
151        and may be a symbolic link.  Do not call os.getcwd() in this
152        case, because it will cause p4 to think that PWD is not inside
153        the client path.
154        """
155
156     os.chdir(path)
157     if not is_client_path:
158         path = os.getcwd()
159     os.environ['PWD'] = path
160
161 def calcDiskFree():
162     """Return free space in bytes on the disk of the given dirname."""
163     if platform.system() == 'Windows':
164         free_bytes = ctypes.c_ulonglong(0)
165         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
166         return free_bytes.value
167     else:
168         st = os.statvfs(os.getcwd())
169         return st.f_bavail * st.f_frsize
170
171 def die(msg):
172     """ Terminate execution. Make sure that any running child processes have been wait()ed for before
173         calling this.
174     """
175     if verbose:
176         raise Exception(msg)
177     else:
178         sys.stderr.write(msg + "\n")
179         sys.exit(1)
180
181 def prompt(prompt_text):
182     """ Prompt the user to choose one of the choices
183
184     Choices are identified in the prompt_text by square brackets around
185     a single letter option.
186     """
187     choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
188     while True:
189         response = raw_input(prompt_text).strip().lower()
190         if not response:
191             continue
192         response = response[0]
193         if response in choices:
194             return response
195
196 def write_pipe(c, stdin):
197     if verbose:
198         sys.stderr.write('Writing pipe: %s\n' % str(c))
199
200     expand = isinstance(c,basestring)
201     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
202     pipe = p.stdin
203     val = pipe.write(stdin)
204     pipe.close()
205     if p.wait():
206         die('Command failed: %s' % str(c))
207
208     return val
209
210 def p4_write_pipe(c, stdin):
211     real_cmd = p4_build_cmd(c)
212     return write_pipe(real_cmd, stdin)
213
214 def read_pipe_full(c):
215     """ Read output from  command. Returns a tuple
216         of the return status, stdout text and stderr
217         text.
218     """
219     if verbose:
220         sys.stderr.write('Reading pipe: %s\n' % str(c))
221
222     expand = isinstance(c,basestring)
223     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
224     (out, err) = p.communicate()
225     return (p.returncode, out, err)
226
227 def read_pipe(c, ignore_error=False):
228     """ Read output from  command. Returns the output text on
229         success. On failure, terminates execution, unless
230         ignore_error is True, when it returns an empty string.
231     """
232     (retcode, out, err) = read_pipe_full(c)
233     if retcode != 0:
234         if ignore_error:
235             out = ""
236         else:
237             die('Command failed: %s\nError: %s' % (str(c), err))
238     return out
239
240 def read_pipe_text(c):
241     """ Read output from a command with trailing whitespace stripped.
242         On error, returns None.
243     """
244     (retcode, out, err) = read_pipe_full(c)
245     if retcode != 0:
246         return None
247     else:
248         return out.rstrip()
249
250 def p4_read_pipe(c, ignore_error=False):
251     real_cmd = p4_build_cmd(c)
252     return read_pipe(real_cmd, ignore_error)
253
254 def read_pipe_lines(c):
255     if verbose:
256         sys.stderr.write('Reading pipe: %s\n' % str(c))
257
258     expand = isinstance(c, basestring)
259     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
260     pipe = p.stdout
261     val = pipe.readlines()
262     if pipe.close() or p.wait():
263         die('Command failed: %s' % str(c))
264
265     return val
266
267 def p4_read_pipe_lines(c):
268     """Specifically invoke p4 on the command supplied. """
269     real_cmd = p4_build_cmd(c)
270     return read_pipe_lines(real_cmd)
271
272 def p4_has_command(cmd):
273     """Ask p4 for help on this command.  If it returns an error, the
274        command does not exist in this version of p4."""
275     real_cmd = p4_build_cmd(["help", cmd])
276     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
277                                    stderr=subprocess.PIPE)
278     p.communicate()
279     return p.returncode == 0
280
281 def p4_has_move_command():
282     """See if the move command exists, that it supports -k, and that
283        it has not been administratively disabled.  The arguments
284        must be correct, but the filenames do not have to exist.  Use
285        ones with wildcards so even if they exist, it will fail."""
286
287     if not p4_has_command("move"):
288         return False
289     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
290     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
291     (out, err) = p.communicate()
292     # return code will be 1 in either case
293     if err.find("Invalid option") >= 0:
294         return False
295     if err.find("disabled") >= 0:
296         return False
297     # assume it failed because @... was invalid changelist
298     return True
299
300 def system(cmd, ignore_error=False):
301     expand = isinstance(cmd,basestring)
302     if verbose:
303         sys.stderr.write("executing %s\n" % str(cmd))
304     retcode = subprocess.call(cmd, shell=expand)
305     if retcode and not ignore_error:
306         raise CalledProcessError(retcode, cmd)
307
308     return retcode
309
310 def p4_system(cmd):
311     """Specifically invoke p4 as the system command. """
312     real_cmd = p4_build_cmd(cmd)
313     expand = isinstance(real_cmd, basestring)
314     retcode = subprocess.call(real_cmd, shell=expand)
315     if retcode:
316         raise CalledProcessError(retcode, real_cmd)
317
318 def die_bad_access(s):
319     die("failure accessing depot: {0}".format(s.rstrip()))
320
321 def p4_check_access(min_expiration=1):
322     """ Check if we can access Perforce - account still logged in
323     """
324     results = p4CmdList(["login", "-s"])
325
326     if len(results) == 0:
327         # should never get here: always get either some results, or a p4ExitCode
328         assert("could not parse response from perforce")
329
330     result = results[0]
331
332     if 'p4ExitCode' in result:
333         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
334         die_bad_access("could not run p4")
335
336     code = result.get("code")
337     if not code:
338         # we get here if we couldn't connect and there was nothing to unmarshal
339         die_bad_access("could not connect")
340
341     elif code == "stat":
342         expiry = result.get("TicketExpiration")
343         if expiry:
344             expiry = int(expiry)
345             if expiry > min_expiration:
346                 # ok to carry on
347                 return
348             else:
349                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
350
351         else:
352             # account without a timeout - all ok
353             return
354
355     elif code == "error":
356         data = result.get("data")
357         if data:
358             die_bad_access("p4 error: {0}".format(data))
359         else:
360             die_bad_access("unknown error")
361     elif code == "info":
362         return
363     else:
364         die_bad_access("unknown error code {0}".format(code))
365
366 _p4_version_string = None
367 def p4_version_string():
368     """Read the version string, showing just the last line, which
369        hopefully is the interesting version bit.
370
371        $ p4 -V
372        Perforce - The Fast Software Configuration Management System.
373        Copyright 1995-2011 Perforce Software.  All rights reserved.
374        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
375     """
376     global _p4_version_string
377     if not _p4_version_string:
378         a = p4_read_pipe_lines(["-V"])
379         _p4_version_string = a[-1].rstrip()
380     return _p4_version_string
381
382 def p4_integrate(src, dest):
383     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
384
385 def p4_sync(f, *options):
386     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
387
388 def p4_add(f):
389     # forcibly add file names with wildcards
390     if wildcard_present(f):
391         p4_system(["add", "-f", f])
392     else:
393         p4_system(["add", f])
394
395 def p4_delete(f):
396     p4_system(["delete", wildcard_encode(f)])
397
398 def p4_edit(f, *options):
399     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
400
401 def p4_revert(f):
402     p4_system(["revert", wildcard_encode(f)])
403
404 def p4_reopen(type, f):
405     p4_system(["reopen", "-t", type, wildcard_encode(f)])
406
407 def p4_reopen_in_change(changelist, files):
408     cmd = ["reopen", "-c", str(changelist)] + files
409     p4_system(cmd)
410
411 def p4_move(src, dest):
412     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
413
414 def p4_last_change():
415     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
416     return int(results[0]['change'])
417
418 def p4_describe(change, shelved=False):
419     """Make sure it returns a valid result by checking for
420        the presence of field "time".  Return a dict of the
421        results."""
422
423     cmd = ["describe", "-s"]
424     if shelved:
425         cmd += ["-S"]
426     cmd += [str(change)]
427
428     ds = p4CmdList(cmd, skip_info=True)
429     if len(ds) != 1:
430         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
431
432     d = ds[0]
433
434     if "p4ExitCode" in d:
435         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
436                                                       str(d)))
437     if "code" in d:
438         if d["code"] == "error":
439             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
440
441     if "time" not in d:
442         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
443
444     return d
445
446 #
447 # Canonicalize the p4 type and return a tuple of the
448 # base type, plus any modifiers.  See "p4 help filetypes"
449 # for a list and explanation.
450 #
451 def split_p4_type(p4type):
452
453     p4_filetypes_historical = {
454         "ctempobj": "binary+Sw",
455         "ctext": "text+C",
456         "cxtext": "text+Cx",
457         "ktext": "text+k",
458         "kxtext": "text+kx",
459         "ltext": "text+F",
460         "tempobj": "binary+FSw",
461         "ubinary": "binary+F",
462         "uresource": "resource+F",
463         "uxbinary": "binary+Fx",
464         "xbinary": "binary+x",
465         "xltext": "text+Fx",
466         "xtempobj": "binary+Swx",
467         "xtext": "text+x",
468         "xunicode": "unicode+x",
469         "xutf16": "utf16+x",
470     }
471     if p4type in p4_filetypes_historical:
472         p4type = p4_filetypes_historical[p4type]
473     mods = ""
474     s = p4type.split("+")
475     base = s[0]
476     mods = ""
477     if len(s) > 1:
478         mods = s[1]
479     return (base, mods)
480
481 #
482 # return the raw p4 type of a file (text, text+ko, etc)
483 #
484 def p4_type(f):
485     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
486     return results[0]['headType']
487
488 #
489 # Given a type base and modifier, return a regexp matching
490 # the keywords that can be expanded in the file
491 #
492 def p4_keywords_regexp_for_type(base, type_mods):
493     if base in ("text", "unicode", "binary"):
494         kwords = None
495         if "ko" in type_mods:
496             kwords = 'Id|Header'
497         elif "k" in type_mods:
498             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
499         else:
500             return None
501         pattern = r"""
502             \$              # Starts with a dollar, followed by...
503             (%s)            # one of the keywords, followed by...
504             (:[^$\n]+)?     # possibly an old expansion, followed by...
505             \$              # another dollar
506             """ % kwords
507         return pattern
508     else:
509         return None
510
511 #
512 # Given a file, return a regexp matching the possible
513 # RCS keywords that will be expanded, or None for files
514 # with kw expansion turned off.
515 #
516 def p4_keywords_regexp_for_file(file):
517     if not os.path.exists(file):
518         return None
519     else:
520         (type_base, type_mods) = split_p4_type(p4_type(file))
521         return p4_keywords_regexp_for_type(type_base, type_mods)
522
523 def setP4ExecBit(file, mode):
524     # Reopens an already open file and changes the execute bit to match
525     # the execute bit setting in the passed in mode.
526
527     p4Type = "+x"
528
529     if not isModeExec(mode):
530         p4Type = getP4OpenedType(file)
531         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
532         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
533         if p4Type[-1] == "+":
534             p4Type = p4Type[0:-1]
535
536     p4_reopen(p4Type, file)
537
538 def getP4OpenedType(file):
539     # Returns the perforce file type for the given file.
540
541     result = p4_read_pipe(["opened", wildcard_encode(file)])
542     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
543     if match:
544         return match.group(1)
545     else:
546         die("Could not determine file type for %s (result: '%s')" % (file, result))
547
548 # Return the set of all p4 labels
549 def getP4Labels(depotPaths):
550     labels = set()
551     if isinstance(depotPaths,basestring):
552         depotPaths = [depotPaths]
553
554     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
555         label = l['label']
556         labels.add(label)
557
558     return labels
559
560 # Return the set of all git tags
561 def getGitTags():
562     gitTags = set()
563     for line in read_pipe_lines(["git", "tag"]):
564         tag = line.strip()
565         gitTags.add(tag)
566     return gitTags
567
568 def diffTreePattern():
569     # This is a simple generator for the diff tree regex pattern. This could be
570     # a class variable if this and parseDiffTreeEntry were a part of a class.
571     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
572     while True:
573         yield pattern
574
575 def parseDiffTreeEntry(entry):
576     """Parses a single diff tree entry into its component elements.
577
578     See git-diff-tree(1) manpage for details about the format of the diff
579     output. This method returns a dictionary with the following elements:
580
581     src_mode - The mode of the source file
582     dst_mode - The mode of the destination file
583     src_sha1 - The sha1 for the source file
584     dst_sha1 - The sha1 fr the destination file
585     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
586     status_score - The score for the status (applicable for 'C' and 'R'
587                    statuses). This is None if there is no score.
588     src - The path for the source file.
589     dst - The path for the destination file. This is only present for
590           copy or renames. If it is not present, this is None.
591
592     If the pattern is not matched, None is returned."""
593
594     match = diffTreePattern().next().match(entry)
595     if match:
596         return {
597             'src_mode': match.group(1),
598             'dst_mode': match.group(2),
599             'src_sha1': match.group(3),
600             'dst_sha1': match.group(4),
601             'status': match.group(5),
602             'status_score': match.group(6),
603             'src': match.group(7),
604             'dst': match.group(10)
605         }
606     return None
607
608 def isModeExec(mode):
609     # Returns True if the given git mode represents an executable file,
610     # otherwise False.
611     return mode[-3:] == "755"
612
613 class P4Exception(Exception):
614     """ Base class for exceptions from the p4 client """
615     def __init__(self, exit_code):
616         self.p4ExitCode = exit_code
617
618 class P4ServerException(P4Exception):
619     """ Base class for exceptions where we get some kind of marshalled up result from the server """
620     def __init__(self, exit_code, p4_result):
621         super(P4ServerException, self).__init__(exit_code)
622         self.p4_result = p4_result
623         self.code = p4_result[0]['code']
624         self.data = p4_result[0]['data']
625
626 class P4RequestSizeException(P4ServerException):
627     """ One of the maxresults or maxscanrows errors """
628     def __init__(self, exit_code, p4_result, limit):
629         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
630         self.limit = limit
631
632 class P4CommandException(P4Exception):
633     """ Something went wrong calling p4 which means we have to give up """
634     def __init__(self, msg):
635         self.msg = msg
636
637     def __str__(self):
638         return self.msg
639
640 def isModeExecChanged(src_mode, dst_mode):
641     return isModeExec(src_mode) != isModeExec(dst_mode)
642
643 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
644         errors_as_exceptions=False):
645
646     if isinstance(cmd,basestring):
647         cmd = "-G " + cmd
648         expand = True
649     else:
650         cmd = ["-G"] + cmd
651         expand = False
652
653     cmd = p4_build_cmd(cmd)
654     if verbose:
655         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
656
657     # Use a temporary file to avoid deadlocks without
658     # subprocess.communicate(), which would put another copy
659     # of stdout into memory.
660     stdin_file = None
661     if stdin is not None:
662         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
663         if isinstance(stdin,basestring):
664             stdin_file.write(stdin)
665         else:
666             for i in stdin:
667                 stdin_file.write(i + '\n')
668         stdin_file.flush()
669         stdin_file.seek(0)
670
671     p4 = subprocess.Popen(cmd,
672                           shell=expand,
673                           stdin=stdin_file,
674                           stdout=subprocess.PIPE)
675
676     result = []
677     try:
678         while True:
679             entry = marshal.load(p4.stdout)
680             if skip_info:
681                 if 'code' in entry and entry['code'] == 'info':
682                     continue
683             if cb is not None:
684                 cb(entry)
685             else:
686                 result.append(entry)
687     except EOFError:
688         pass
689     exitCode = p4.wait()
690     if exitCode != 0:
691         if errors_as_exceptions:
692             if len(result) > 0:
693                 data = result[0].get('data')
694                 if data:
695                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
696                     if not m:
697                         m = re.search('Request too large \(over (\d+)\)', data)
698
699                     if m:
700                         limit = int(m.group(1))
701                         raise P4RequestSizeException(exitCode, result, limit)
702
703                 raise P4ServerException(exitCode, result)
704             else:
705                 raise P4Exception(exitCode)
706         else:
707             entry = {}
708             entry["p4ExitCode"] = exitCode
709             result.append(entry)
710
711     return result
712
713 def p4Cmd(cmd):
714     list = p4CmdList(cmd)
715     result = {}
716     for entry in list:
717         result.update(entry)
718     return result;
719
720 def p4Where(depotPath):
721     if not depotPath.endswith("/"):
722         depotPath += "/"
723     depotPathLong = depotPath + "..."
724     outputList = p4CmdList(["where", depotPathLong])
725     output = None
726     for entry in outputList:
727         if "depotFile" in entry:
728             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
729             # The base path always ends with "/...".
730             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
731                 output = entry
732                 break
733         elif "data" in entry:
734             data = entry.get("data")
735             space = data.find(" ")
736             if data[:space] == depotPath:
737                 output = entry
738                 break
739     if output == None:
740         return ""
741     if output["code"] == "error":
742         return ""
743     clientPath = ""
744     if "path" in output:
745         clientPath = output.get("path")
746     elif "data" in output:
747         data = output.get("data")
748         lastSpace = data.rfind(" ")
749         clientPath = data[lastSpace + 1:]
750
751     if clientPath.endswith("..."):
752         clientPath = clientPath[:-3]
753     return clientPath
754
755 def currentGitBranch():
756     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
757
758 def isValidGitDir(path):
759     return git_dir(path) != None
760
761 def parseRevision(ref):
762     return read_pipe("git rev-parse %s" % ref).strip()
763
764 def branchExists(ref):
765     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
766                      ignore_error=True)
767     return len(rev) > 0
768
769 def extractLogMessageFromGitCommit(commit):
770     logMessage = ""
771
772     ## fixme: title is first line of commit, not 1st paragraph.
773     foundTitle = False
774     for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
775        if not foundTitle:
776            if len(log) == 1:
777                foundTitle = True
778            continue
779
780        logMessage += log
781     return logMessage
782
783 def extractSettingsGitLog(log):
784     values = {}
785     for line in log.split("\n"):
786         line = line.strip()
787         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
788         if not m:
789             continue
790
791         assignments = m.group(1).split (':')
792         for a in assignments:
793             vals = a.split ('=')
794             key = vals[0].strip()
795             val = ('='.join (vals[1:])).strip()
796             if val.endswith ('\"') and val.startswith('"'):
797                 val = val[1:-1]
798
799             values[key] = val
800
801     paths = values.get("depot-paths")
802     if not paths:
803         paths = values.get("depot-path")
804     if paths:
805         values['depot-paths'] = paths.split(',')
806     return values
807
808 def gitBranchExists(branch):
809     proc = subprocess.Popen(["git", "rev-parse", branch],
810                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
811     return proc.wait() == 0;
812
813 def gitUpdateRef(ref, newvalue):
814     subprocess.check_call(["git", "update-ref", ref, newvalue])
815
816 def gitDeleteRef(ref):
817     subprocess.check_call(["git", "update-ref", "-d", ref])
818
819 _gitConfig = {}
820
821 def gitConfig(key, typeSpecifier=None):
822     if key not in _gitConfig:
823         cmd = [ "git", "config" ]
824         if typeSpecifier:
825             cmd += [ typeSpecifier ]
826         cmd += [ key ]
827         s = read_pipe(cmd, ignore_error=True)
828         _gitConfig[key] = s.strip()
829     return _gitConfig[key]
830
831 def gitConfigBool(key):
832     """Return a bool, using git config --bool.  It is True only if the
833        variable is set to true, and False if set to false or not present
834        in the config."""
835
836     if key not in _gitConfig:
837         _gitConfig[key] = gitConfig(key, '--bool') == "true"
838     return _gitConfig[key]
839
840 def gitConfigInt(key):
841     if key not in _gitConfig:
842         cmd = [ "git", "config", "--int", key ]
843         s = read_pipe(cmd, ignore_error=True)
844         v = s.strip()
845         try:
846             _gitConfig[key] = int(gitConfig(key, '--int'))
847         except ValueError:
848             _gitConfig[key] = None
849     return _gitConfig[key]
850
851 def gitConfigList(key):
852     if key not in _gitConfig:
853         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
854         _gitConfig[key] = s.strip().splitlines()
855         if _gitConfig[key] == ['']:
856             _gitConfig[key] = []
857     return _gitConfig[key]
858
859 def p4BranchesInGit(branchesAreInRemotes=True):
860     """Find all the branches whose names start with "p4/", looking
861        in remotes or heads as specified by the argument.  Return
862        a dictionary of { branch: revision } for each one found.
863        The branch names are the short names, without any
864        "p4/" prefix."""
865
866     branches = {}
867
868     cmdline = "git rev-parse --symbolic "
869     if branchesAreInRemotes:
870         cmdline += "--remotes"
871     else:
872         cmdline += "--branches"
873
874     for line in read_pipe_lines(cmdline):
875         line = line.strip()
876
877         # only import to p4/
878         if not line.startswith('p4/'):
879             continue
880         # special symbolic ref to p4/master
881         if line == "p4/HEAD":
882             continue
883
884         # strip off p4/ prefix
885         branch = line[len("p4/"):]
886
887         branches[branch] = parseRevision(line)
888
889     return branches
890
891 def branch_exists(branch):
892     """Make sure that the given ref name really exists."""
893
894     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
895     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
896     out, _ = p.communicate()
897     if p.returncode:
898         return False
899     # expect exactly one line of output: the branch name
900     return out.rstrip() == branch
901
902 def findUpstreamBranchPoint(head = "HEAD"):
903     branches = p4BranchesInGit()
904     # map from depot-path to branch name
905     branchByDepotPath = {}
906     for branch in branches.keys():
907         tip = branches[branch]
908         log = extractLogMessageFromGitCommit(tip)
909         settings = extractSettingsGitLog(log)
910         if "depot-paths" in settings:
911             paths = ",".join(settings["depot-paths"])
912             branchByDepotPath[paths] = "remotes/p4/" + branch
913
914     settings = None
915     parent = 0
916     while parent < 65535:
917         commit = head + "~%s" % parent
918         log = extractLogMessageFromGitCommit(commit)
919         settings = extractSettingsGitLog(log)
920         if "depot-paths" in settings:
921             paths = ",".join(settings["depot-paths"])
922             if paths in branchByDepotPath:
923                 return [branchByDepotPath[paths], settings]
924
925         parent = parent + 1
926
927     return ["", settings]
928
929 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
930     if not silent:
931         print("Creating/updating branch(es) in %s based on origin branch(es)"
932                % localRefPrefix)
933
934     originPrefix = "origin/p4/"
935
936     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
937         line = line.strip()
938         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
939             continue
940
941         headName = line[len(originPrefix):]
942         remoteHead = localRefPrefix + headName
943         originHead = line
944
945         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
946         if ('depot-paths' not in original
947             or 'change' not in original):
948             continue
949
950         update = False
951         if not gitBranchExists(remoteHead):
952             if verbose:
953                 print("creating %s" % remoteHead)
954             update = True
955         else:
956             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
957             if 'change' in settings:
958                 if settings['depot-paths'] == original['depot-paths']:
959                     originP4Change = int(original['change'])
960                     p4Change = int(settings['change'])
961                     if originP4Change > p4Change:
962                         print("%s (%s) is newer than %s (%s). "
963                                "Updating p4 branch from origin."
964                                % (originHead, originP4Change,
965                                   remoteHead, p4Change))
966                         update = True
967                 else:
968                     print("Ignoring: %s was imported from %s while "
969                            "%s was imported from %s"
970                            % (originHead, ','.join(original['depot-paths']),
971                               remoteHead, ','.join(settings['depot-paths'])))
972
973         if update:
974             system("git update-ref %s %s" % (remoteHead, originHead))
975
976 def originP4BranchesExist():
977         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
978
979
980 def p4ParseNumericChangeRange(parts):
981     changeStart = int(parts[0][1:])
982     if parts[1] == '#head':
983         changeEnd = p4_last_change()
984     else:
985         changeEnd = int(parts[1])
986
987     return (changeStart, changeEnd)
988
989 def chooseBlockSize(blockSize):
990     if blockSize:
991         return blockSize
992     else:
993         return defaultBlockSize
994
995 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
996     assert depotPaths
997
998     # Parse the change range into start and end. Try to find integer
999     # revision ranges as these can be broken up into blocks to avoid
1000     # hitting server-side limits (maxrows, maxscanresults). But if
1001     # that doesn't work, fall back to using the raw revision specifier
1002     # strings, without using block mode.
1003
1004     if changeRange is None or changeRange == '':
1005         changeStart = 1
1006         changeEnd = p4_last_change()
1007         block_size = chooseBlockSize(requestedBlockSize)
1008     else:
1009         parts = changeRange.split(',')
1010         assert len(parts) == 2
1011         try:
1012             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
1013             block_size = chooseBlockSize(requestedBlockSize)
1014         except ValueError:
1015             changeStart = parts[0][1:]
1016             changeEnd = parts[1]
1017             if requestedBlockSize:
1018                 die("cannot use --changes-block-size with non-numeric revisions")
1019             block_size = None
1020
1021     changes = set()
1022
1023     # Retrieve changes a block at a time, to prevent running
1024     # into a MaxResults/MaxScanRows error from the server. If
1025     # we _do_ hit one of those errors, turn down the block size
1026
1027     while True:
1028         cmd = ['changes']
1029
1030         if block_size:
1031             end = min(changeEnd, changeStart + block_size)
1032             revisionRange = "%d,%d" % (changeStart, end)
1033         else:
1034             revisionRange = "%s,%s" % (changeStart, changeEnd)
1035
1036         for p in depotPaths:
1037             cmd += ["%s...@%s" % (p, revisionRange)]
1038
1039         # fetch the changes
1040         try:
1041             result = p4CmdList(cmd, errors_as_exceptions=True)
1042         except P4RequestSizeException as e:
1043             if not block_size:
1044                 block_size = e.limit
1045             elif block_size > e.limit:
1046                 block_size = e.limit
1047             else:
1048                 block_size = max(2, block_size // 2)
1049
1050             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1051             continue
1052         except P4Exception as e:
1053             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1054
1055         # Insert changes in chronological order
1056         for entry in reversed(result):
1057             if 'change' not in entry:
1058                 continue
1059             changes.add(int(entry['change']))
1060
1061         if not block_size:
1062             break
1063
1064         if end >= changeEnd:
1065             break
1066
1067         changeStart = end + 1
1068
1069     changes = sorted(changes)
1070     return changes
1071
1072 def p4PathStartsWith(path, prefix):
1073     # This method tries to remedy a potential mixed-case issue:
1074     #
1075     # If UserA adds  //depot/DirA/file1
1076     # and UserB adds //depot/dira/file2
1077     #
1078     # we may or may not have a problem. If you have core.ignorecase=true,
1079     # we treat DirA and dira as the same directory
1080     if gitConfigBool("core.ignorecase"):
1081         return path.lower().startswith(prefix.lower())
1082     return path.startswith(prefix)
1083
1084 def getClientSpec():
1085     """Look at the p4 client spec, create a View() object that contains
1086        all the mappings, and return it."""
1087
1088     specList = p4CmdList("client -o")
1089     if len(specList) != 1:
1090         die('Output from "client -o" is %d lines, expecting 1' %
1091             len(specList))
1092
1093     # dictionary of all client parameters
1094     entry = specList[0]
1095
1096     # the //client/ name
1097     client_name = entry["Client"]
1098
1099     # just the keys that start with "View"
1100     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1101
1102     # hold this new View
1103     view = View(client_name)
1104
1105     # append the lines, in order, to the view
1106     for view_num in range(len(view_keys)):
1107         k = "View%d" % view_num
1108         if k not in view_keys:
1109             die("Expected view key %s missing" % k)
1110         view.append(entry[k])
1111
1112     return view
1113
1114 def getClientRoot():
1115     """Grab the client directory."""
1116
1117     output = p4CmdList("client -o")
1118     if len(output) != 1:
1119         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1120
1121     entry = output[0]
1122     if "Root" not in entry:
1123         die('Client has no "Root"')
1124
1125     return entry["Root"]
1126
1127 #
1128 # P4 wildcards are not allowed in filenames.  P4 complains
1129 # if you simply add them, but you can force it with "-f", in
1130 # which case it translates them into %xx encoding internally.
1131 #
1132 def wildcard_decode(path):
1133     # Search for and fix just these four characters.  Do % last so
1134     # that fixing it does not inadvertently create new %-escapes.
1135     # Cannot have * in a filename in windows; untested as to
1136     # what p4 would do in such a case.
1137     if not platform.system() == "Windows":
1138         path = path.replace("%2A", "*")
1139     path = path.replace("%23", "#") \
1140                .replace("%40", "@") \
1141                .replace("%25", "%")
1142     return path
1143
1144 def wildcard_encode(path):
1145     # do % first to avoid double-encoding the %s introduced here
1146     path = path.replace("%", "%25") \
1147                .replace("*", "%2A") \
1148                .replace("#", "%23") \
1149                .replace("@", "%40")
1150     return path
1151
1152 def wildcard_present(path):
1153     m = re.search("[*#@%]", path)
1154     return m is not None
1155
1156 class LargeFileSystem(object):
1157     """Base class for large file system support."""
1158
1159     def __init__(self, writeToGitStream):
1160         self.largeFiles = set()
1161         self.writeToGitStream = writeToGitStream
1162
1163     def generatePointer(self, cloneDestination, contentFile):
1164         """Return the content of a pointer file that is stored in Git instead of
1165            the actual content."""
1166         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1167
1168     def pushFile(self, localLargeFile):
1169         """Push the actual content which is not stored in the Git repository to
1170            a server."""
1171         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1172
1173     def hasLargeFileExtension(self, relPath):
1174         return reduce(
1175             lambda a, b: a or b,
1176             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1177             False
1178         )
1179
1180     def generateTempFile(self, contents):
1181         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1182         for d in contents:
1183             contentFile.write(d)
1184         contentFile.close()
1185         return contentFile.name
1186
1187     def exceedsLargeFileThreshold(self, relPath, contents):
1188         if gitConfigInt('git-p4.largeFileThreshold'):
1189             contentsSize = sum(len(d) for d in contents)
1190             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1191                 return True
1192         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1193             contentsSize = sum(len(d) for d in contents)
1194             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1195                 return False
1196             contentTempFile = self.generateTempFile(contents)
1197             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1198             with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1199                 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1200                 compressedContentsSize = zf.infolist()[0].compress_size
1201             os.remove(contentTempFile)
1202             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1203                 return True
1204         return False
1205
1206     def addLargeFile(self, relPath):
1207         self.largeFiles.add(relPath)
1208
1209     def removeLargeFile(self, relPath):
1210         self.largeFiles.remove(relPath)
1211
1212     def isLargeFile(self, relPath):
1213         return relPath in self.largeFiles
1214
1215     def processContent(self, git_mode, relPath, contents):
1216         """Processes the content of git fast import. This method decides if a
1217            file is stored in the large file system and handles all necessary
1218            steps."""
1219         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1220             contentTempFile = self.generateTempFile(contents)
1221             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1222             if pointer_git_mode:
1223                 git_mode = pointer_git_mode
1224             if localLargeFile:
1225                 # Move temp file to final location in large file system
1226                 largeFileDir = os.path.dirname(localLargeFile)
1227                 if not os.path.isdir(largeFileDir):
1228                     os.makedirs(largeFileDir)
1229                 shutil.move(contentTempFile, localLargeFile)
1230                 self.addLargeFile(relPath)
1231                 if gitConfigBool('git-p4.largeFilePush'):
1232                     self.pushFile(localLargeFile)
1233                 if verbose:
1234                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1235         return (git_mode, contents)
1236
1237 class MockLFS(LargeFileSystem):
1238     """Mock large file system for testing."""
1239
1240     def generatePointer(self, contentFile):
1241         """The pointer content is the original content prefixed with "pointer-".
1242            The local filename of the large file storage is derived from the file content.
1243            """
1244         with open(contentFile, 'r') as f:
1245             content = next(f)
1246             gitMode = '100644'
1247             pointerContents = 'pointer-' + content
1248             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1249             return (gitMode, pointerContents, localLargeFile)
1250
1251     def pushFile(self, localLargeFile):
1252         """The remote filename of the large file storage is the same as the local
1253            one but in a different directory.
1254            """
1255         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1256         if not os.path.exists(remotePath):
1257             os.makedirs(remotePath)
1258         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1259
1260 class GitLFS(LargeFileSystem):
1261     """Git LFS as backend for the git-p4 large file system.
1262        See https://git-lfs.github.com/ for details."""
1263
1264     def __init__(self, *args):
1265         LargeFileSystem.__init__(self, *args)
1266         self.baseGitAttributes = []
1267
1268     def generatePointer(self, contentFile):
1269         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1270            mode and content which is stored in the Git repository instead of
1271            the actual content. Return also the new location of the actual
1272            content.
1273            """
1274         if os.path.getsize(contentFile) == 0:
1275             return (None, '', None)
1276
1277         pointerProcess = subprocess.Popen(
1278             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1279             stdout=subprocess.PIPE
1280         )
1281         pointerFile = pointerProcess.stdout.read()
1282         if pointerProcess.wait():
1283             os.remove(contentFile)
1284             die('git-lfs pointer command failed. Did you install the extension?')
1285
1286         # Git LFS removed the preamble in the output of the 'pointer' command
1287         # starting from version 1.2.0. Check for the preamble here to support
1288         # earlier versions.
1289         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1290         if pointerFile.startswith('Git LFS pointer for'):
1291             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1292
1293         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1294         # if someone use external lfs.storage ( not in local repo git )
1295         lfs_path = gitConfig('lfs.storage')
1296         if not lfs_path:
1297             lfs_path = 'lfs'
1298         if not os.path.isabs(lfs_path):
1299             lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1300         localLargeFile = os.path.join(
1301             lfs_path,
1302             'objects', oid[:2], oid[2:4],
1303             oid,
1304         )
1305         # LFS Spec states that pointer files should not have the executable bit set.
1306         gitMode = '100644'
1307         return (gitMode, pointerFile, localLargeFile)
1308
1309     def pushFile(self, localLargeFile):
1310         uploadProcess = subprocess.Popen(
1311             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1312         )
1313         if uploadProcess.wait():
1314             die('git-lfs push command failed. Did you define a remote?')
1315
1316     def generateGitAttributes(self):
1317         return (
1318             self.baseGitAttributes +
1319             [
1320                 '\n',
1321                 '#\n',
1322                 '# Git LFS (see https://git-lfs.github.com/)\n',
1323                 '#\n',
1324             ] +
1325             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1326                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1327             ] +
1328             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1329                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1330             ]
1331         )
1332
1333     def addLargeFile(self, relPath):
1334         LargeFileSystem.addLargeFile(self, relPath)
1335         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1336
1337     def removeLargeFile(self, relPath):
1338         LargeFileSystem.removeLargeFile(self, relPath)
1339         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1340
1341     def processContent(self, git_mode, relPath, contents):
1342         if relPath == '.gitattributes':
1343             self.baseGitAttributes = contents
1344             return (git_mode, self.generateGitAttributes())
1345         else:
1346             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1347
1348 class Command:
1349     delete_actions = ( "delete", "move/delete", "purge" )
1350     add_actions = ( "add", "branch", "move/add" )
1351
1352     def __init__(self):
1353         self.usage = "usage: %prog [options]"
1354         self.needsGit = True
1355         self.verbose = False
1356
1357     # This is required for the "append" update_shelve action
1358     def ensure_value(self, attr, value):
1359         if not hasattr(self, attr) or getattr(self, attr) is None:
1360             setattr(self, attr, value)
1361         return getattr(self, attr)
1362
1363 class P4UserMap:
1364     def __init__(self):
1365         self.userMapFromPerforceServer = False
1366         self.myP4UserId = None
1367
1368     def p4UserId(self):
1369         if self.myP4UserId:
1370             return self.myP4UserId
1371
1372         results = p4CmdList("user -o")
1373         for r in results:
1374             if 'User' in r:
1375                 self.myP4UserId = r['User']
1376                 return r['User']
1377         die("Could not find your p4 user id")
1378
1379     def p4UserIsMe(self, p4User):
1380         # return True if the given p4 user is actually me
1381         me = self.p4UserId()
1382         if not p4User or p4User != me:
1383             return False
1384         else:
1385             return True
1386
1387     def getUserCacheFilename(self):
1388         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1389         return home + "/.gitp4-usercache.txt"
1390
1391     def getUserMapFromPerforceServer(self):
1392         if self.userMapFromPerforceServer:
1393             return
1394         self.users = {}
1395         self.emails = {}
1396
1397         for output in p4CmdList("users"):
1398             if "User" not in output:
1399                 continue
1400             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1401             self.emails[output["Email"]] = output["User"]
1402
1403         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1404         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1405             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1406             if mapUser and len(mapUser[0]) == 3:
1407                 user = mapUser[0][0]
1408                 fullname = mapUser[0][1]
1409                 email = mapUser[0][2]
1410                 self.users[user] = fullname + " <" + email + ">"
1411                 self.emails[email] = user
1412
1413         s = ''
1414         for (key, val) in self.users.items():
1415             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1416
1417         open(self.getUserCacheFilename(), "wb").write(s)
1418         self.userMapFromPerforceServer = True
1419
1420     def loadUserMapFromCache(self):
1421         self.users = {}
1422         self.userMapFromPerforceServer = False
1423         try:
1424             cache = open(self.getUserCacheFilename(), "rb")
1425             lines = cache.readlines()
1426             cache.close()
1427             for line in lines:
1428                 entry = line.strip().split("\t")
1429                 self.users[entry[0]] = entry[1]
1430         except IOError:
1431             self.getUserMapFromPerforceServer()
1432
1433 class P4Debug(Command):
1434     def __init__(self):
1435         Command.__init__(self)
1436         self.options = []
1437         self.description = "A tool to debug the output of p4 -G."
1438         self.needsGit = False
1439
1440     def run(self, args):
1441         j = 0
1442         for output in p4CmdList(args):
1443             print('Element: %d' % j)
1444             j += 1
1445             print(output)
1446         return True
1447
1448 class P4RollBack(Command):
1449     def __init__(self):
1450         Command.__init__(self)
1451         self.options = [
1452             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1453         ]
1454         self.description = "A tool to debug the multi-branch import. Don't use :)"
1455         self.rollbackLocalBranches = False
1456
1457     def run(self, args):
1458         if len(args) != 1:
1459             return False
1460         maxChange = int(args[0])
1461
1462         if "p4ExitCode" in p4Cmd("changes -m 1"):
1463             die("Problems executing p4");
1464
1465         if self.rollbackLocalBranches:
1466             refPrefix = "refs/heads/"
1467             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1468         else:
1469             refPrefix = "refs/remotes/"
1470             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1471
1472         for line in lines:
1473             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1474                 line = line.strip()
1475                 ref = refPrefix + line
1476                 log = extractLogMessageFromGitCommit(ref)
1477                 settings = extractSettingsGitLog(log)
1478
1479                 depotPaths = settings['depot-paths']
1480                 change = settings['change']
1481
1482                 changed = False
1483
1484                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1485                                                            for p in depotPaths]))) == 0:
1486                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1487                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1488                     continue
1489
1490                 while change and int(change) > maxChange:
1491                     changed = True
1492                     if self.verbose:
1493                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1494                     system("git update-ref %s \"%s^\"" % (ref, ref))
1495                     log = extractLogMessageFromGitCommit(ref)
1496                     settings =  extractSettingsGitLog(log)
1497
1498
1499                     depotPaths = settings['depot-paths']
1500                     change = settings['change']
1501
1502                 if changed:
1503                     print("%s rewound to %s" % (ref, change))
1504
1505         return True
1506
1507 class P4Submit(Command, P4UserMap):
1508
1509     conflict_behavior_choices = ("ask", "skip", "quit")
1510
1511     def __init__(self):
1512         Command.__init__(self)
1513         P4UserMap.__init__(self)
1514         self.options = [
1515                 optparse.make_option("--origin", dest="origin"),
1516                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1517                 # preserve the user, requires relevant p4 permissions
1518                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1519                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1520                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1521                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1522                 optparse.make_option("--conflict", dest="conflict_behavior",
1523                                      choices=self.conflict_behavior_choices),
1524                 optparse.make_option("--branch", dest="branch"),
1525                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1526                                      help="Shelve instead of submit. Shelved files are reverted, "
1527                                      "restoring the workspace to the state before the shelve"),
1528                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1529                                      metavar="CHANGELIST",
1530                                      help="update an existing shelved changelist, implies --shelve, "
1531                                            "repeat in-order for multiple shelved changelists"),
1532                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1533                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1534                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1535                                      help="Disable rebase after submit is completed. Can be useful if you "
1536                                      "work from a local git branch that is not master"),
1537                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1538                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1539         ]
1540         self.description = """Submit changes from git to the perforce depot.\n
1541     The `p4-pre-submit` hook is executed if it exists and is executable.
1542     The hook takes no parameters and nothing from standard input. Exiting with
1543     non-zero status from this script prevents `git-p4 submit` from launching.
1544
1545     One usage scenario is to run unit tests in the hook."""
1546
1547         self.usage += " [name of git branch to submit into perforce depot]"
1548         self.origin = ""
1549         self.detectRenames = False
1550         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1551         self.dry_run = False
1552         self.shelve = False
1553         self.update_shelve = list()
1554         self.commit = ""
1555         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1556         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1557         self.prepare_p4_only = False
1558         self.conflict_behavior = None
1559         self.isWindows = (platform.system() == "Windows")
1560         self.exportLabels = False
1561         self.p4HasMoveCommand = p4_has_move_command()
1562         self.branch = None
1563
1564         if gitConfig('git-p4.largeFileSystem'):
1565             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1566
1567     def check(self):
1568         if len(p4CmdList("opened ...")) > 0:
1569             die("You have files opened with perforce! Close them before starting the sync.")
1570
1571     def separate_jobs_from_description(self, message):
1572         """Extract and return a possible Jobs field in the commit
1573            message.  It goes into a separate section in the p4 change
1574            specification.
1575
1576            A jobs line starts with "Jobs:" and looks like a new field
1577            in a form.  Values are white-space separated on the same
1578            line or on following lines that start with a tab.
1579
1580            This does not parse and extract the full git commit message
1581            like a p4 form.  It just sees the Jobs: line as a marker
1582            to pass everything from then on directly into the p4 form,
1583            but outside the description section.
1584
1585            Return a tuple (stripped log message, jobs string)."""
1586
1587         m = re.search(r'^Jobs:', message, re.MULTILINE)
1588         if m is None:
1589             return (message, None)
1590
1591         jobtext = message[m.start():]
1592         stripped_message = message[:m.start()].rstrip()
1593         return (stripped_message, jobtext)
1594
1595     def prepareLogMessage(self, template, message, jobs):
1596         """Edits the template returned from "p4 change -o" to insert
1597            the message in the Description field, and the jobs text in
1598            the Jobs field."""
1599         result = ""
1600
1601         inDescriptionSection = False
1602
1603         for line in template.split("\n"):
1604             if line.startswith("#"):
1605                 result += line + "\n"
1606                 continue
1607
1608             if inDescriptionSection:
1609                 if line.startswith("Files:") or line.startswith("Jobs:"):
1610                     inDescriptionSection = False
1611                     # insert Jobs section
1612                     if jobs:
1613                         result += jobs + "\n"
1614                 else:
1615                     continue
1616             else:
1617                 if line.startswith("Description:"):
1618                     inDescriptionSection = True
1619                     line += "\n"
1620                     for messageLine in message.split("\n"):
1621                         line += "\t" + messageLine + "\n"
1622
1623             result += line + "\n"
1624
1625         return result
1626
1627     def patchRCSKeywords(self, file, pattern):
1628         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1629         (handle, outFileName) = tempfile.mkstemp(dir='.')
1630         try:
1631             outFile = os.fdopen(handle, "w+")
1632             inFile = open(file, "r")
1633             regexp = re.compile(pattern, re.VERBOSE)
1634             for line in inFile.readlines():
1635                 line = regexp.sub(r'$\1$', line)
1636                 outFile.write(line)
1637             inFile.close()
1638             outFile.close()
1639             # Forcibly overwrite the original file
1640             os.unlink(file)
1641             shutil.move(outFileName, file)
1642         except:
1643             # cleanup our temporary file
1644             os.unlink(outFileName)
1645             print("Failed to strip RCS keywords in %s" % file)
1646             raise
1647
1648         print("Patched up RCS keywords in %s" % file)
1649
1650     def p4UserForCommit(self,id):
1651         # Return the tuple (perforce user,git email) for a given git commit id
1652         self.getUserMapFromPerforceServer()
1653         gitEmail = read_pipe(["git", "log", "--max-count=1",
1654                               "--format=%ae", id])
1655         gitEmail = gitEmail.strip()
1656         if gitEmail not in self.emails:
1657             return (None,gitEmail)
1658         else:
1659             return (self.emails[gitEmail],gitEmail)
1660
1661     def checkValidP4Users(self,commits):
1662         # check if any git authors cannot be mapped to p4 users
1663         for id in commits:
1664             (user,email) = self.p4UserForCommit(id)
1665             if not user:
1666                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1667                 if gitConfigBool("git-p4.allowMissingP4Users"):
1668                     print("%s" % msg)
1669                 else:
1670                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1671
1672     def lastP4Changelist(self):
1673         # Get back the last changelist number submitted in this client spec. This
1674         # then gets used to patch up the username in the change. If the same
1675         # client spec is being used by multiple processes then this might go
1676         # wrong.
1677         results = p4CmdList("client -o")        # find the current client
1678         client = None
1679         for r in results:
1680             if 'Client' in r:
1681                 client = r['Client']
1682                 break
1683         if not client:
1684             die("could not get client spec")
1685         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1686         for r in results:
1687             if 'change' in r:
1688                 return r['change']
1689         die("Could not get changelist number for last submit - cannot patch up user details")
1690
1691     def modifyChangelistUser(self, changelist, newUser):
1692         # fixup the user field of a changelist after it has been submitted.
1693         changes = p4CmdList("change -o %s" % changelist)
1694         if len(changes) != 1:
1695             die("Bad output from p4 change modifying %s to user %s" %
1696                 (changelist, newUser))
1697
1698         c = changes[0]
1699         if c['User'] == newUser: return   # nothing to do
1700         c['User'] = newUser
1701         input = marshal.dumps(c)
1702
1703         result = p4CmdList("change -f -i", stdin=input)
1704         for r in result:
1705             if 'code' in r:
1706                 if r['code'] == 'error':
1707                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1708             if 'data' in r:
1709                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1710                 return
1711         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1712
1713     def canChangeChangelists(self):
1714         # check to see if we have p4 admin or super-user permissions, either of
1715         # which are required to modify changelists.
1716         results = p4CmdList(["protects", self.depotPath])
1717         for r in results:
1718             if 'perm' in r:
1719                 if r['perm'] == 'admin':
1720                     return 1
1721                 if r['perm'] == 'super':
1722                     return 1
1723         return 0
1724
1725     def prepareSubmitTemplate(self, changelist=None):
1726         """Run "p4 change -o" to grab a change specification template.
1727            This does not use "p4 -G", as it is nice to keep the submission
1728            template in original order, since a human might edit it.
1729
1730            Remove lines in the Files section that show changes to files
1731            outside the depot path we're committing into."""
1732
1733         [upstream, settings] = findUpstreamBranchPoint()
1734
1735         template = """\
1736 # A Perforce Change Specification.
1737 #
1738 #  Change:      The change number. 'new' on a new changelist.
1739 #  Date:        The date this specification was last modified.
1740 #  Client:      The client on which the changelist was created.  Read-only.
1741 #  User:        The user who created the changelist.
1742 #  Status:      Either 'pending' or 'submitted'. Read-only.
1743 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1744 #  Description: Comments about the changelist.  Required.
1745 #  Jobs:        What opened jobs are to be closed by this changelist.
1746 #               You may delete jobs from this list.  (New changelists only.)
1747 #  Files:       What opened files from the default changelist are to be added
1748 #               to this changelist.  You may delete files from this list.
1749 #               (New changelists only.)
1750 """
1751         files_list = []
1752         inFilesSection = False
1753         change_entry = None
1754         args = ['change', '-o']
1755         if changelist:
1756             args.append(str(changelist))
1757         for entry in p4CmdList(args):
1758             if 'code' not in entry:
1759                 continue
1760             if entry['code'] == 'stat':
1761                 change_entry = entry
1762                 break
1763         if not change_entry:
1764             die('Failed to decode output of p4 change -o')
1765         for key, value in change_entry.iteritems():
1766             if key.startswith('File'):
1767                 if 'depot-paths' in settings:
1768                     if not [p for p in settings['depot-paths']
1769                             if p4PathStartsWith(value, p)]:
1770                         continue
1771                 else:
1772                     if not p4PathStartsWith(value, self.depotPath):
1773                         continue
1774                 files_list.append(value)
1775                 continue
1776         # Output in the order expected by prepareLogMessage
1777         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1778             if key not in change_entry:
1779                 continue
1780             template += '\n'
1781             template += key + ':'
1782             if key == 'Description':
1783                 template += '\n'
1784             for field_line in change_entry[key].splitlines():
1785                 template += '\t'+field_line+'\n'
1786         if len(files_list) > 0:
1787             template += '\n'
1788             template += 'Files:\n'
1789         for path in files_list:
1790             template += '\t'+path+'\n'
1791         return template
1792
1793     def edit_template(self, template_file):
1794         """Invoke the editor to let the user change the submission
1795            message.  Return true if okay to continue with the submit."""
1796
1797         # if configured to skip the editing part, just submit
1798         if gitConfigBool("git-p4.skipSubmitEdit"):
1799             return True
1800
1801         # look at the modification time, to check later if the user saved
1802         # the file
1803         mtime = os.stat(template_file).st_mtime
1804
1805         # invoke the editor
1806         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1807             editor = os.environ.get("P4EDITOR")
1808         else:
1809             editor = read_pipe("git var GIT_EDITOR").strip()
1810         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1811
1812         # If the file was not saved, prompt to see if this patch should
1813         # be skipped.  But skip this verification step if configured so.
1814         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1815             return True
1816
1817         # modification time updated means user saved the file
1818         if os.stat(template_file).st_mtime > mtime:
1819             return True
1820
1821         response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1822         if response == 'y':
1823             return True
1824         if response == 'n':
1825             return False
1826
1827     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1828         # diff
1829         if "P4DIFF" in os.environ:
1830             del(os.environ["P4DIFF"])
1831         diff = ""
1832         for editedFile in editedFiles:
1833             diff += p4_read_pipe(['diff', '-du',
1834                                   wildcard_encode(editedFile)])
1835
1836         # new file diff
1837         newdiff = ""
1838         for newFile in filesToAdd:
1839             newdiff += "==== new file ====\n"
1840             newdiff += "--- /dev/null\n"
1841             newdiff += "+++ %s\n" % newFile
1842
1843             is_link = os.path.islink(newFile)
1844             expect_link = newFile in symlinks
1845
1846             if is_link and expect_link:
1847                 newdiff += "+%s\n" % os.readlink(newFile)
1848             else:
1849                 f = open(newFile, "r")
1850                 for line in f.readlines():
1851                     newdiff += "+" + line
1852                 f.close()
1853
1854         return (diff + newdiff).replace('\r\n', '\n')
1855
1856     def applyCommit(self, id):
1857         """Apply one commit, return True if it succeeded."""
1858
1859         print("Applying", read_pipe(["git", "show", "-s",
1860                                      "--format=format:%h %s", id]))
1861
1862         (p4User, gitEmail) = self.p4UserForCommit(id)
1863
1864         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1865         filesToAdd = set()
1866         filesToChangeType = set()
1867         filesToDelete = set()
1868         editedFiles = set()
1869         pureRenameCopy = set()
1870         symlinks = set()
1871         filesToChangeExecBit = {}
1872         all_files = list()
1873
1874         for line in diff:
1875             diff = parseDiffTreeEntry(line)
1876             modifier = diff['status']
1877             path = diff['src']
1878             all_files.append(path)
1879
1880             if modifier == "M":
1881                 p4_edit(path)
1882                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1883                     filesToChangeExecBit[path] = diff['dst_mode']
1884                 editedFiles.add(path)
1885             elif modifier == "A":
1886                 filesToAdd.add(path)
1887                 filesToChangeExecBit[path] = diff['dst_mode']
1888                 if path in filesToDelete:
1889                     filesToDelete.remove(path)
1890
1891                 dst_mode = int(diff['dst_mode'], 8)
1892                 if dst_mode == 0o120000:
1893                     symlinks.add(path)
1894
1895             elif modifier == "D":
1896                 filesToDelete.add(path)
1897                 if path in filesToAdd:
1898                     filesToAdd.remove(path)
1899             elif modifier == "C":
1900                 src, dest = diff['src'], diff['dst']
1901                 all_files.append(dest)
1902                 p4_integrate(src, dest)
1903                 pureRenameCopy.add(dest)
1904                 if diff['src_sha1'] != diff['dst_sha1']:
1905                     p4_edit(dest)
1906                     pureRenameCopy.discard(dest)
1907                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1908                     p4_edit(dest)
1909                     pureRenameCopy.discard(dest)
1910                     filesToChangeExecBit[dest] = diff['dst_mode']
1911                 if self.isWindows:
1912                     # turn off read-only attribute
1913                     os.chmod(dest, stat.S_IWRITE)
1914                 os.unlink(dest)
1915                 editedFiles.add(dest)
1916             elif modifier == "R":
1917                 src, dest = diff['src'], diff['dst']
1918                 all_files.append(dest)
1919                 if self.p4HasMoveCommand:
1920                     p4_edit(src)        # src must be open before move
1921                     p4_move(src, dest)  # opens for (move/delete, move/add)
1922                 else:
1923                     p4_integrate(src, dest)
1924                     if diff['src_sha1'] != diff['dst_sha1']:
1925                         p4_edit(dest)
1926                     else:
1927                         pureRenameCopy.add(dest)
1928                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1929                     if not self.p4HasMoveCommand:
1930                         p4_edit(dest)   # with move: already open, writable
1931                     filesToChangeExecBit[dest] = diff['dst_mode']
1932                 if not self.p4HasMoveCommand:
1933                     if self.isWindows:
1934                         os.chmod(dest, stat.S_IWRITE)
1935                     os.unlink(dest)
1936                     filesToDelete.add(src)
1937                 editedFiles.add(dest)
1938             elif modifier == "T":
1939                 filesToChangeType.add(path)
1940             else:
1941                 die("unknown modifier %s for %s" % (modifier, path))
1942
1943         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1944         patchcmd = diffcmd + " | git apply "
1945         tryPatchCmd = patchcmd + "--check -"
1946         applyPatchCmd = patchcmd + "--check --apply -"
1947         patch_succeeded = True
1948
1949         if os.system(tryPatchCmd) != 0:
1950             fixed_rcs_keywords = False
1951             patch_succeeded = False
1952             print("Unfortunately applying the change failed!")
1953
1954             # Patch failed, maybe it's just RCS keyword woes. Look through
1955             # the patch to see if that's possible.
1956             if gitConfigBool("git-p4.attemptRCSCleanup"):
1957                 file = None
1958                 pattern = None
1959                 kwfiles = {}
1960                 for file in editedFiles | filesToDelete:
1961                     # did this file's delta contain RCS keywords?
1962                     pattern = p4_keywords_regexp_for_file(file)
1963
1964                     if pattern:
1965                         # this file is a possibility...look for RCS keywords.
1966                         regexp = re.compile(pattern, re.VERBOSE)
1967                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1968                             if regexp.search(line):
1969                                 if verbose:
1970                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
1971                                 kwfiles[file] = pattern
1972                                 break
1973
1974                 for file in kwfiles:
1975                     if verbose:
1976                         print("zapping %s with %s" % (line,pattern))
1977                     # File is being deleted, so not open in p4.  Must
1978                     # disable the read-only bit on windows.
1979                     if self.isWindows and file not in editedFiles:
1980                         os.chmod(file, stat.S_IWRITE)
1981                     self.patchRCSKeywords(file, kwfiles[file])
1982                     fixed_rcs_keywords = True
1983
1984             if fixed_rcs_keywords:
1985                 print("Retrying the patch with RCS keywords cleaned up")
1986                 if os.system(tryPatchCmd) == 0:
1987                     patch_succeeded = True
1988
1989         if not patch_succeeded:
1990             for f in editedFiles:
1991                 p4_revert(f)
1992             return False
1993
1994         #
1995         # Apply the patch for real, and do add/delete/+x handling.
1996         #
1997         system(applyPatchCmd)
1998
1999         for f in filesToChangeType:
2000             p4_edit(f, "-t", "auto")
2001         for f in filesToAdd:
2002             p4_add(f)
2003         for f in filesToDelete:
2004             p4_revert(f)
2005             p4_delete(f)
2006
2007         # Set/clear executable bits
2008         for f in filesToChangeExecBit.keys():
2009             mode = filesToChangeExecBit[f]
2010             setP4ExecBit(f, mode)
2011
2012         update_shelve = 0
2013         if len(self.update_shelve) > 0:
2014             update_shelve = self.update_shelve.pop(0)
2015             p4_reopen_in_change(update_shelve, all_files)
2016
2017         #
2018         # Build p4 change description, starting with the contents
2019         # of the git commit message.
2020         #
2021         logMessage = extractLogMessageFromGitCommit(id)
2022         logMessage = logMessage.strip()
2023         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
2024
2025         template = self.prepareSubmitTemplate(update_shelve)
2026         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
2027
2028         if self.preserveUser:
2029            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
2030
2031         if self.checkAuthorship and not self.p4UserIsMe(p4User):
2032             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2033             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2034             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2035
2036         separatorLine = "######## everything below this line is just the diff #######\n"
2037         if not self.prepare_p4_only:
2038             submitTemplate += separatorLine
2039             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2040
2041         (handle, fileName) = tempfile.mkstemp()
2042         tmpFile = os.fdopen(handle, "w+b")
2043         if self.isWindows:
2044             submitTemplate = submitTemplate.replace("\n", "\r\n")
2045         tmpFile.write(submitTemplate)
2046         tmpFile.close()
2047
2048         if self.prepare_p4_only:
2049             #
2050             # Leave the p4 tree prepared, and the submit template around
2051             # and let the user decide what to do next
2052             #
2053             print()
2054             print("P4 workspace prepared for submission.")
2055             print("To submit or revert, go to client workspace")
2056             print("  " + self.clientPath)
2057             print()
2058             print("To submit, use \"p4 submit\" to write a new description,")
2059             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2060                   " \"git p4\"." % fileName)
2061             print("You can delete the file \"%s\" when finished." % fileName)
2062
2063             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2064                 print("To preserve change ownership by user %s, you must\n" \
2065                       "do \"p4 change -f <change>\" after submitting and\n" \
2066                       "edit the User field.")
2067             if pureRenameCopy:
2068                 print("After submitting, renamed files must be re-synced.")
2069                 print("Invoke \"p4 sync -f\" on each of these files:")
2070                 for f in pureRenameCopy:
2071                     print("  " + f)
2072
2073             print()
2074             print("To revert the changes, use \"p4 revert ...\", and delete")
2075             print("the submit template file \"%s\"" % fileName)
2076             if filesToAdd:
2077                 print("Since the commit adds new files, they must be deleted:")
2078                 for f in filesToAdd:
2079                     print("  " + f)
2080             print()
2081             return True
2082
2083         #
2084         # Let the user edit the change description, then submit it.
2085         #
2086         submitted = False
2087
2088         try:
2089             if self.edit_template(fileName):
2090                 # read the edited message and submit
2091                 tmpFile = open(fileName, "rb")
2092                 message = tmpFile.read()
2093                 tmpFile.close()
2094                 if self.isWindows:
2095                     message = message.replace("\r\n", "\n")
2096                 submitTemplate = message[:message.index(separatorLine)]
2097
2098                 if update_shelve:
2099                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2100                 elif self.shelve:
2101                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2102                 else:
2103                     p4_write_pipe(['submit', '-i'], submitTemplate)
2104                     # The rename/copy happened by applying a patch that created a
2105                     # new file.  This leaves it writable, which confuses p4.
2106                     for f in pureRenameCopy:
2107                         p4_sync(f, "-f")
2108
2109                 if self.preserveUser:
2110                     if p4User:
2111                         # Get last changelist number. Cannot easily get it from
2112                         # the submit command output as the output is
2113                         # unmarshalled.
2114                         changelist = self.lastP4Changelist()
2115                         self.modifyChangelistUser(changelist, p4User)
2116
2117                 submitted = True
2118
2119         finally:
2120             # skip this patch
2121             if not submitted or self.shelve:
2122                 if self.shelve:
2123                     print ("Reverting shelved files.")
2124                 else:
2125                     print ("Submission cancelled, undoing p4 changes.")
2126                 for f in editedFiles | filesToDelete:
2127                     p4_revert(f)
2128                 for f in filesToAdd:
2129                     p4_revert(f)
2130                     os.remove(f)
2131
2132         os.remove(fileName)
2133         return submitted
2134
2135     # Export git tags as p4 labels. Create a p4 label and then tag
2136     # with that.
2137     def exportGitTags(self, gitTags):
2138         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2139         if len(validLabelRegexp) == 0:
2140             validLabelRegexp = defaultLabelRegexp
2141         m = re.compile(validLabelRegexp)
2142
2143         for name in gitTags:
2144
2145             if not m.match(name):
2146                 if verbose:
2147                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2148                 continue
2149
2150             # Get the p4 commit this corresponds to
2151             logMessage = extractLogMessageFromGitCommit(name)
2152             values = extractSettingsGitLog(logMessage)
2153
2154             if 'change' not in values:
2155                 # a tag pointing to something not sent to p4; ignore
2156                 if verbose:
2157                     print("git tag %s does not give a p4 commit" % name)
2158                 continue
2159             else:
2160                 changelist = values['change']
2161
2162             # Get the tag details.
2163             inHeader = True
2164             isAnnotated = False
2165             body = []
2166             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2167                 l = l.strip()
2168                 if inHeader:
2169                     if re.match(r'tag\s+', l):
2170                         isAnnotated = True
2171                     elif re.match(r'\s*$', l):
2172                         inHeader = False
2173                         continue
2174                 else:
2175                     body.append(l)
2176
2177             if not isAnnotated:
2178                 body = ["lightweight tag imported by git p4\n"]
2179
2180             # Create the label - use the same view as the client spec we are using
2181             clientSpec = getClientSpec()
2182
2183             labelTemplate  = "Label: %s\n" % name
2184             labelTemplate += "Description:\n"
2185             for b in body:
2186                 labelTemplate += "\t" + b + "\n"
2187             labelTemplate += "View:\n"
2188             for depot_side in clientSpec.mappings:
2189                 labelTemplate += "\t%s\n" % depot_side
2190
2191             if self.dry_run:
2192                 print("Would create p4 label %s for tag" % name)
2193             elif self.prepare_p4_only:
2194                 print("Not creating p4 label %s for tag due to option" \
2195                       " --prepare-p4-only" % name)
2196             else:
2197                 p4_write_pipe(["label", "-i"], labelTemplate)
2198
2199                 # Use the label
2200                 p4_system(["tag", "-l", name] +
2201                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2202
2203                 if verbose:
2204                     print("created p4 label for tag %s" % name)
2205
2206     def run(self, args):
2207         if len(args) == 0:
2208             self.master = currentGitBranch()
2209         elif len(args) == 1:
2210             self.master = args[0]
2211             if not branchExists(self.master):
2212                 die("Branch %s does not exist" % self.master)
2213         else:
2214             return False
2215
2216         for i in self.update_shelve:
2217             if i <= 0:
2218                 sys.exit("invalid changelist %d" % i)
2219
2220         if self.master:
2221             allowSubmit = gitConfig("git-p4.allowSubmit")
2222             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2223                 die("%s is not in git-p4.allowSubmit" % self.master)
2224
2225         [upstream, settings] = findUpstreamBranchPoint()
2226         self.depotPath = settings['depot-paths'][0]
2227         if len(self.origin) == 0:
2228             self.origin = upstream
2229
2230         if len(self.update_shelve) > 0:
2231             self.shelve = True
2232
2233         if self.preserveUser:
2234             if not self.canChangeChangelists():
2235                 die("Cannot preserve user names without p4 super-user or admin permissions")
2236
2237         # if not set from the command line, try the config file
2238         if self.conflict_behavior is None:
2239             val = gitConfig("git-p4.conflict")
2240             if val:
2241                 if val not in self.conflict_behavior_choices:
2242                     die("Invalid value '%s' for config git-p4.conflict" % val)
2243             else:
2244                 val = "ask"
2245             self.conflict_behavior = val
2246
2247         if self.verbose:
2248             print("Origin branch is " + self.origin)
2249
2250         if len(self.depotPath) == 0:
2251             print("Internal error: cannot locate perforce depot path from existing branches")
2252             sys.exit(128)
2253
2254         self.useClientSpec = False
2255         if gitConfigBool("git-p4.useclientspec"):
2256             self.useClientSpec = True
2257         if self.useClientSpec:
2258             self.clientSpecDirs = getClientSpec()
2259
2260         # Check for the existence of P4 branches
2261         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2262
2263         if self.useClientSpec and not branchesDetected:
2264             # all files are relative to the client spec
2265             self.clientPath = getClientRoot()
2266         else:
2267             self.clientPath = p4Where(self.depotPath)
2268
2269         if self.clientPath == "":
2270             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2271
2272         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2273         self.oldWorkingDirectory = os.getcwd()
2274
2275         # ensure the clientPath exists
2276         new_client_dir = False
2277         if not os.path.exists(self.clientPath):
2278             new_client_dir = True
2279             os.makedirs(self.clientPath)
2280
2281         chdir(self.clientPath, is_client_path=True)
2282         if self.dry_run:
2283             print("Would synchronize p4 checkout in %s" % self.clientPath)
2284         else:
2285             print("Synchronizing p4 checkout...")
2286             if new_client_dir:
2287                 # old one was destroyed, and maybe nobody told p4
2288                 p4_sync("...", "-f")
2289             else:
2290                 p4_sync("...")
2291         self.check()
2292
2293         commits = []
2294         if self.master:
2295             committish = self.master
2296         else:
2297             committish = 'HEAD'
2298
2299         if self.commit != "":
2300             if self.commit.find("..") != -1:
2301                 limits_ish = self.commit.split("..")
2302                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2303                     commits.append(line.strip())
2304                 commits.reverse()
2305             else:
2306                 commits.append(self.commit)
2307         else:
2308             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2309                 commits.append(line.strip())
2310             commits.reverse()
2311
2312         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2313             self.checkAuthorship = False
2314         else:
2315             self.checkAuthorship = True
2316
2317         if self.preserveUser:
2318             self.checkValidP4Users(commits)
2319
2320         #
2321         # Build up a set of options to be passed to diff when
2322         # submitting each commit to p4.
2323         #
2324         if self.detectRenames:
2325             # command-line -M arg
2326             self.diffOpts = "-M"
2327         else:
2328             # If not explicitly set check the config variable
2329             detectRenames = gitConfig("git-p4.detectRenames")
2330
2331             if detectRenames.lower() == "false" or detectRenames == "":
2332                 self.diffOpts = ""
2333             elif detectRenames.lower() == "true":
2334                 self.diffOpts = "-M"
2335             else:
2336                 self.diffOpts = "-M%s" % detectRenames
2337
2338         # no command-line arg for -C or --find-copies-harder, just
2339         # config variables
2340         detectCopies = gitConfig("git-p4.detectCopies")
2341         if detectCopies.lower() == "false" or detectCopies == "":
2342             pass
2343         elif detectCopies.lower() == "true":
2344             self.diffOpts += " -C"
2345         else:
2346             self.diffOpts += " -C%s" % detectCopies
2347
2348         if gitConfigBool("git-p4.detectCopiesHarder"):
2349             self.diffOpts += " --find-copies-harder"
2350
2351         num_shelves = len(self.update_shelve)
2352         if num_shelves > 0 and num_shelves != len(commits):
2353             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2354                      (len(commits), num_shelves))
2355
2356         hooks_path = gitConfig("core.hooksPath")
2357         if len(hooks_path) <= 0:
2358             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2359
2360         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2361         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2362             sys.exit(1)
2363
2364         #
2365         # Apply the commits, one at a time.  On failure, ask if should
2366         # continue to try the rest of the patches, or quit.
2367         #
2368         if self.dry_run:
2369             print("Would apply")
2370         applied = []
2371         last = len(commits) - 1
2372         for i, commit in enumerate(commits):
2373             if self.dry_run:
2374                 print(" ", read_pipe(["git", "show", "-s",
2375                                       "--format=format:%h %s", commit]))
2376                 ok = True
2377             else:
2378                 ok = self.applyCommit(commit)
2379             if ok:
2380                 applied.append(commit)
2381             else:
2382                 if self.prepare_p4_only and i < last:
2383                     print("Processing only the first commit due to option" \
2384                           " --prepare-p4-only")
2385                     break
2386                 if i < last:
2387                     # prompt for what to do, or use the option/variable
2388                     if self.conflict_behavior == "ask":
2389                         print("What do you want to do?")
2390                         response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2391                     elif self.conflict_behavior == "skip":
2392                         response = "s"
2393                     elif self.conflict_behavior == "quit":
2394                         response = "q"
2395                     else:
2396                         die("Unknown conflict_behavior '%s'" %
2397                             self.conflict_behavior)
2398
2399                     if response == "s":
2400                         print("Skipping this commit, but applying the rest")
2401                     if response == "q":
2402                         print("Quitting")
2403                         break
2404
2405         chdir(self.oldWorkingDirectory)
2406         shelved_applied = "shelved" if self.shelve else "applied"
2407         if self.dry_run:
2408             pass
2409         elif self.prepare_p4_only:
2410             pass
2411         elif len(commits) == len(applied):
2412             print("All commits {0}!".format(shelved_applied))
2413
2414             sync = P4Sync()
2415             if self.branch:
2416                 sync.branch = self.branch
2417             if self.disable_p4sync:
2418                 sync.sync_origin_only()
2419             else:
2420                 sync.run([])
2421
2422                 if not self.disable_rebase:
2423                     rebase = P4Rebase()
2424                     rebase.rebase()
2425
2426         else:
2427             if len(applied) == 0:
2428                 print("No commits {0}.".format(shelved_applied))
2429             else:
2430                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2431                 for c in commits:
2432                     if c in applied:
2433                         star = "*"
2434                     else:
2435                         star = " "
2436                     print(star, read_pipe(["git", "show", "-s",
2437                                            "--format=format:%h %s",  c]))
2438                 print("You will have to do 'git p4 sync' and rebase.")
2439
2440         if gitConfigBool("git-p4.exportLabels"):
2441             self.exportLabels = True
2442
2443         if self.exportLabels:
2444             p4Labels = getP4Labels(self.depotPath)
2445             gitTags = getGitTags()
2446
2447             missingGitTags = gitTags - p4Labels
2448             self.exportGitTags(missingGitTags)
2449
2450         # exit with error unless everything applied perfectly
2451         if len(commits) != len(applied):
2452                 sys.exit(1)
2453
2454         return True
2455
2456 class View(object):
2457     """Represent a p4 view ("p4 help views"), and map files in a
2458        repo according to the view."""
2459
2460     def __init__(self, client_name):
2461         self.mappings = []
2462         self.client_prefix = "//%s/" % client_name
2463         # cache results of "p4 where" to lookup client file locations
2464         self.client_spec_path_cache = {}
2465
2466     def append(self, view_line):
2467         """Parse a view line, splitting it into depot and client
2468            sides.  Append to self.mappings, preserving order.  This
2469            is only needed for tag creation."""
2470
2471         # Split the view line into exactly two words.  P4 enforces
2472         # structure on these lines that simplifies this quite a bit.
2473         #
2474         # Either or both words may be double-quoted.
2475         # Single quotes do not matter.
2476         # Double-quote marks cannot occur inside the words.
2477         # A + or - prefix is also inside the quotes.
2478         # There are no quotes unless they contain a space.
2479         # The line is already white-space stripped.
2480         # The two words are separated by a single space.
2481         #
2482         if view_line[0] == '"':
2483             # First word is double quoted.  Find its end.
2484             close_quote_index = view_line.find('"', 1)
2485             if close_quote_index <= 0:
2486                 die("No first-word closing quote found: %s" % view_line)
2487             depot_side = view_line[1:close_quote_index]
2488             # skip closing quote and space
2489             rhs_index = close_quote_index + 1 + 1
2490         else:
2491             space_index = view_line.find(" ")
2492             if space_index <= 0:
2493                 die("No word-splitting space found: %s" % view_line)
2494             depot_side = view_line[0:space_index]
2495             rhs_index = space_index + 1
2496
2497         # prefix + means overlay on previous mapping
2498         if depot_side.startswith("+"):
2499             depot_side = depot_side[1:]
2500
2501         # prefix - means exclude this path, leave out of mappings
2502         exclude = False
2503         if depot_side.startswith("-"):
2504             exclude = True
2505             depot_side = depot_side[1:]
2506
2507         if not exclude:
2508             self.mappings.append(depot_side)
2509
2510     def convert_client_path(self, clientFile):
2511         # chop off //client/ part to make it relative
2512         if not clientFile.startswith(self.client_prefix):
2513             die("No prefix '%s' on clientFile '%s'" %
2514                 (self.client_prefix, clientFile))
2515         return clientFile[len(self.client_prefix):]
2516
2517     def update_client_spec_path_cache(self, files):
2518         """ Caching file paths by "p4 where" batch query """
2519
2520         # List depot file paths exclude that already cached
2521         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2522
2523         if len(fileArgs) == 0:
2524             return  # All files in cache
2525
2526         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2527         for res in where_result:
2528             if "code" in res and res["code"] == "error":
2529                 # assume error is "... file(s) not in client view"
2530                 continue
2531             if "clientFile" not in res:
2532                 die("No clientFile in 'p4 where' output")
2533             if "unmap" in res:
2534                 # it will list all of them, but only one not unmap-ped
2535                 continue
2536             if gitConfigBool("core.ignorecase"):
2537                 res['depotFile'] = res['depotFile'].lower()
2538             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2539
2540         # not found files or unmap files set to ""
2541         for depotFile in fileArgs:
2542             if gitConfigBool("core.ignorecase"):
2543                 depotFile = depotFile.lower()
2544             if depotFile not in self.client_spec_path_cache:
2545                 self.client_spec_path_cache[depotFile] = ""
2546
2547     def map_in_client(self, depot_path):
2548         """Return the relative location in the client where this
2549            depot file should live.  Returns "" if the file should
2550            not be mapped in the client."""
2551
2552         if gitConfigBool("core.ignorecase"):
2553             depot_path = depot_path.lower()
2554
2555         if depot_path in self.client_spec_path_cache:
2556             return self.client_spec_path_cache[depot_path]
2557
2558         die( "Error: %s is not found in client spec path" % depot_path )
2559         return ""
2560
2561 def cloneExcludeCallback(option, opt_str, value, parser):
2562     # prepend "/" because the first "/" was consumed as part of the option itself.
2563     # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2564     parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2565
2566 class P4Sync(Command, P4UserMap):
2567
2568     def __init__(self):
2569         Command.__init__(self)
2570         P4UserMap.__init__(self)
2571         self.options = [
2572                 optparse.make_option("--branch", dest="branch"),
2573                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2574                 optparse.make_option("--changesfile", dest="changesFile"),
2575                 optparse.make_option("--silent", dest="silent", action="store_true"),
2576                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2577                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2578                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2579                                      help="Import into refs/heads/ , not refs/remotes"),
2580                 optparse.make_option("--max-changes", dest="maxChanges",
2581                                      help="Maximum number of changes to import"),
2582                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2583                                      help="Internal block size to use when iteratively calling p4 changes"),
2584                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2585                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2586                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2587                                      help="Only sync files that are included in the Perforce Client Spec"),
2588                 optparse.make_option("-/", dest="cloneExclude",
2589                                      action="callback", callback=cloneExcludeCallback, type="string",
2590                                      help="exclude depot path"),
2591         ]
2592         self.description = """Imports from Perforce into a git repository.\n
2593     example:
2594     //depot/my/project/ -- to import the current head
2595     //depot/my/project/@all -- to import everything
2596     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2597
2598     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2599
2600         self.usage += " //depot/path[@revRange]"
2601         self.silent = False
2602         self.createdBranches = set()
2603         self.committedChanges = set()
2604         self.branch = ""
2605         self.detectBranches = False
2606         self.detectLabels = False
2607         self.importLabels = False
2608         self.changesFile = ""
2609         self.syncWithOrigin = True
2610         self.importIntoRemotes = True
2611         self.maxChanges = ""
2612         self.changes_block_size = None
2613         self.keepRepoPath = False
2614         self.depotPaths = None
2615         self.p4BranchesInGit = []
2616         self.cloneExclude = []
2617         self.useClientSpec = False
2618         self.useClientSpec_from_options = False
2619         self.clientSpecDirs = None
2620         self.tempBranches = []
2621         self.tempBranchLocation = "refs/git-p4-tmp"
2622         self.largeFileSystem = None
2623         self.suppress_meta_comment = False
2624
2625         if gitConfig('git-p4.largeFileSystem'):
2626             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2627             self.largeFileSystem = largeFileSystemConstructor(
2628                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2629             )
2630
2631         if gitConfig("git-p4.syncFromOrigin") == "false":
2632             self.syncWithOrigin = False
2633
2634         self.depotPaths = []
2635         self.changeRange = ""
2636         self.previousDepotPaths = []
2637         self.hasOrigin = False
2638
2639         # map from branch depot path to parent branch
2640         self.knownBranches = {}
2641         self.initialParents = {}
2642
2643         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2644         self.labels = {}
2645
2646     # Force a checkpoint in fast-import and wait for it to finish
2647     def checkpoint(self):
2648         self.gitStream.write("checkpoint\n\n")
2649         self.gitStream.write("progress checkpoint\n\n")
2650         out = self.gitOutput.readline()
2651         if self.verbose:
2652             print("checkpoint finished: " + out)
2653
2654     def isPathWanted(self, path):
2655         for p in self.cloneExclude:
2656             if p.endswith("/"):
2657                 if p4PathStartsWith(path, p):
2658                     return False
2659             # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2660             elif path.lower() == p.lower():
2661                 return False
2662         for p in self.depotPaths:
2663             if p4PathStartsWith(path, p):
2664                 return True
2665         return False
2666
2667     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2668         files = []
2669         fnum = 0
2670         while "depotFile%s" % fnum in commit:
2671             path =  commit["depotFile%s" % fnum]
2672             found = self.isPathWanted(path)
2673             if not found:
2674                 fnum = fnum + 1
2675                 continue
2676
2677             file = {}
2678             file["path"] = path
2679             file["rev"] = commit["rev%s" % fnum]
2680             file["action"] = commit["action%s" % fnum]
2681             file["type"] = commit["type%s" % fnum]
2682             if shelved:
2683                 file["shelved_cl"] = int(shelved_cl)
2684             files.append(file)
2685             fnum = fnum + 1
2686         return files
2687
2688     def extractJobsFromCommit(self, commit):
2689         jobs = []
2690         jnum = 0
2691         while "job%s" % jnum in commit:
2692             job = commit["job%s" % jnum]
2693             jobs.append(job)
2694             jnum = jnum + 1
2695         return jobs
2696
2697     def stripRepoPath(self, path, prefixes):
2698         """When streaming files, this is called to map a p4 depot path
2699            to where it should go in git.  The prefixes are either
2700            self.depotPaths, or self.branchPrefixes in the case of
2701            branch detection."""
2702
2703         if self.useClientSpec:
2704             # branch detection moves files up a level (the branch name)
2705             # from what client spec interpretation gives
2706             path = self.clientSpecDirs.map_in_client(path)
2707             if self.detectBranches:
2708                 for b in self.knownBranches:
2709                     if p4PathStartsWith(path, b + "/"):
2710                         path = path[len(b)+1:]
2711
2712         elif self.keepRepoPath:
2713             # Preserve everything in relative path name except leading
2714             # //depot/; just look at first prefix as they all should
2715             # be in the same depot.
2716             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2717             if p4PathStartsWith(path, depot):
2718                 path = path[len(depot):]
2719
2720         else:
2721             for p in prefixes:
2722                 if p4PathStartsWith(path, p):
2723                     path = path[len(p):]
2724                     break
2725
2726         path = wildcard_decode(path)
2727         return path
2728
2729     def splitFilesIntoBranches(self, commit):
2730         """Look at each depotFile in the commit to figure out to what
2731            branch it belongs."""
2732
2733         if self.clientSpecDirs:
2734             files = self.extractFilesFromCommit(commit)
2735             self.clientSpecDirs.update_client_spec_path_cache(files)
2736
2737         branches = {}
2738         fnum = 0
2739         while "depotFile%s" % fnum in commit:
2740             path =  commit["depotFile%s" % fnum]
2741             found = self.isPathWanted(path)
2742             if not found:
2743                 fnum = fnum + 1
2744                 continue
2745
2746             file = {}
2747             file["path"] = path
2748             file["rev"] = commit["rev%s" % fnum]
2749             file["action"] = commit["action%s" % fnum]
2750             file["type"] = commit["type%s" % fnum]
2751             fnum = fnum + 1
2752
2753             # start with the full relative path where this file would
2754             # go in a p4 client
2755             if self.useClientSpec:
2756                 relPath = self.clientSpecDirs.map_in_client(path)
2757             else:
2758                 relPath = self.stripRepoPath(path, self.depotPaths)
2759
2760             for branch in self.knownBranches.keys():
2761                 # add a trailing slash so that a commit into qt/4.2foo
2762                 # doesn't end up in qt/4.2, e.g.
2763                 if p4PathStartsWith(relPath, branch + "/"):
2764                     if branch not in branches:
2765                         branches[branch] = []
2766                     branches[branch].append(file)
2767                     break
2768
2769         return branches
2770
2771     def writeToGitStream(self, gitMode, relPath, contents):
2772         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2773         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2774         for d in contents:
2775             self.gitStream.write(d)
2776         self.gitStream.write('\n')
2777
2778     def encodeWithUTF8(self, path):
2779         try:
2780             path.decode('ascii')
2781         except:
2782             encoding = 'utf8'
2783             if gitConfig('git-p4.pathEncoding'):
2784                 encoding = gitConfig('git-p4.pathEncoding')
2785             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2786             if self.verbose:
2787                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2788         return path
2789
2790     # output one file from the P4 stream
2791     # - helper for streamP4Files
2792
2793     def streamOneP4File(self, file, contents):
2794         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2795         relPath = self.encodeWithUTF8(relPath)
2796         if verbose:
2797             if 'fileSize' in self.stream_file:
2798                 size = int(self.stream_file['fileSize'])
2799             else:
2800                 size = 0 # deleted files don't get a fileSize apparently
2801             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2802             sys.stdout.flush()
2803
2804         (type_base, type_mods) = split_p4_type(file["type"])
2805
2806         git_mode = "100644"
2807         if "x" in type_mods:
2808             git_mode = "100755"
2809         if type_base == "symlink":
2810             git_mode = "120000"
2811             # p4 print on a symlink sometimes contains "target\n";
2812             # if it does, remove the newline
2813             data = ''.join(contents)
2814             if not data:
2815                 # Some version of p4 allowed creating a symlink that pointed
2816                 # to nothing.  This causes p4 errors when checking out such
2817                 # a change, and errors here too.  Work around it by ignoring
2818                 # the bad symlink; hopefully a future change fixes it.
2819                 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2820                 return
2821             elif data[-1] == '\n':
2822                 contents = [data[:-1]]
2823             else:
2824                 contents = [data]
2825
2826         if type_base == "utf16":
2827             # p4 delivers different text in the python output to -G
2828             # than it does when using "print -o", or normal p4 client
2829             # operations.  utf16 is converted to ascii or utf8, perhaps.
2830             # But ascii text saved as -t utf16 is completely mangled.
2831             # Invoke print -o to get the real contents.
2832             #
2833             # On windows, the newlines will always be mangled by print, so put
2834             # them back too.  This is not needed to the cygwin windows version,
2835             # just the native "NT" type.
2836             #
2837             try:
2838                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2839             except Exception as e:
2840                 if 'Translation of file content failed' in str(e):
2841                     type_base = 'binary'
2842                 else:
2843                     raise e
2844             else:
2845                 if p4_version_string().find('/NT') >= 0:
2846                     text = text.replace('\r\n', '\n')
2847                 contents = [ text ]
2848
2849         if type_base == "apple":
2850             # Apple filetype files will be streamed as a concatenation of
2851             # its appledouble header and the contents.  This is useless
2852             # on both macs and non-macs.  If using "print -q -o xx", it
2853             # will create "xx" with the data, and "%xx" with the header.
2854             # This is also not very useful.
2855             #
2856             # Ideally, someday, this script can learn how to generate
2857             # appledouble files directly and import those to git, but
2858             # non-mac machines can never find a use for apple filetype.
2859             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2860             return
2861
2862         # Note that we do not try to de-mangle keywords on utf16 files,
2863         # even though in theory somebody may want that.
2864         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2865         if pattern:
2866             regexp = re.compile(pattern, re.VERBOSE)
2867             text = ''.join(contents)
2868             text = regexp.sub(r'$\1$', text)
2869             contents = [ text ]
2870
2871         if self.largeFileSystem:
2872             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2873
2874         self.writeToGitStream(git_mode, relPath, contents)
2875
2876     def streamOneP4Deletion(self, file):
2877         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2878         relPath = self.encodeWithUTF8(relPath)
2879         if verbose:
2880             sys.stdout.write("delete %s\n" % relPath)
2881             sys.stdout.flush()
2882         self.gitStream.write("D %s\n" % relPath)
2883
2884         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2885             self.largeFileSystem.removeLargeFile(relPath)
2886
2887     # handle another chunk of streaming data
2888     def streamP4FilesCb(self, marshalled):
2889
2890         # catch p4 errors and complain
2891         err = None
2892         if "code" in marshalled:
2893             if marshalled["code"] == "error":
2894                 if "data" in marshalled:
2895                     err = marshalled["data"].rstrip()
2896
2897         if not err and 'fileSize' in self.stream_file:
2898             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2899             if required_bytes > 0:
2900                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2901                     os.getcwd(), required_bytes/1024/1024
2902                 )
2903
2904         if err:
2905             f = None
2906             if self.stream_have_file_info:
2907                 if "depotFile" in self.stream_file:
2908                     f = self.stream_file["depotFile"]
2909             # force a failure in fast-import, else an empty
2910             # commit will be made
2911             self.gitStream.write("\n")
2912             self.gitStream.write("die-now\n")
2913             self.gitStream.close()
2914             # ignore errors, but make sure it exits first
2915             self.importProcess.wait()
2916             if f:
2917                 die("Error from p4 print for %s: %s" % (f, err))
2918             else:
2919                 die("Error from p4 print: %s" % err)
2920
2921         if 'depotFile' in marshalled and self.stream_have_file_info:
2922             # start of a new file - output the old one first
2923             self.streamOneP4File(self.stream_file, self.stream_contents)
2924             self.stream_file = {}
2925             self.stream_contents = []
2926             self.stream_have_file_info = False
2927
2928         # pick up the new file information... for the
2929         # 'data' field we need to append to our array
2930         for k in marshalled.keys():
2931             if k == 'data':
2932                 if 'streamContentSize' not in self.stream_file:
2933                     self.stream_file['streamContentSize'] = 0
2934                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2935                 self.stream_contents.append(marshalled['data'])
2936             else:
2937                 self.stream_file[k] = marshalled[k]
2938
2939         if (verbose and
2940             'streamContentSize' in self.stream_file and
2941             'fileSize' in self.stream_file and
2942             'depotFile' in self.stream_file):
2943             size = int(self.stream_file["fileSize"])
2944             if size > 0:
2945                 progress = 100*self.stream_file['streamContentSize']/size
2946                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2947                 sys.stdout.flush()
2948
2949         self.stream_have_file_info = True
2950
2951     # Stream directly from "p4 files" into "git fast-import"
2952     def streamP4Files(self, files):
2953         filesForCommit = []
2954         filesToRead = []
2955         filesToDelete = []
2956
2957         for f in files:
2958             filesForCommit.append(f)
2959             if f['action'] in self.delete_actions:
2960                 filesToDelete.append(f)
2961             else:
2962                 filesToRead.append(f)
2963
2964         # deleted files...
2965         for f in filesToDelete:
2966             self.streamOneP4Deletion(f)
2967
2968         if len(filesToRead) > 0:
2969             self.stream_file = {}
2970             self.stream_contents = []
2971             self.stream_have_file_info = False
2972
2973             # curry self argument
2974             def streamP4FilesCbSelf(entry):
2975                 self.streamP4FilesCb(entry)
2976
2977             fileArgs = []
2978             for f in filesToRead:
2979                 if 'shelved_cl' in f:
2980                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2981                     # the contents
2982                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2983                 else:
2984                     fileArg = '%s#%s' % (f['path'], f['rev'])
2985
2986                 fileArgs.append(fileArg)
2987
2988             p4CmdList(["-x", "-", "print"],
2989                       stdin=fileArgs,
2990                       cb=streamP4FilesCbSelf)
2991
2992             # do the last chunk
2993             if 'depotFile' in self.stream_file:
2994                 self.streamOneP4File(self.stream_file, self.stream_contents)
2995
2996     def make_email(self, userid):
2997         if userid in self.users:
2998             return self.users[userid]
2999         else:
3000             return "%s <a@b>" % userid
3001
3002     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
3003         """ Stream a p4 tag.
3004         commit is either a git commit, or a fast-import mark, ":<p4commit>"
3005         """
3006
3007         if verbose:
3008             print("writing tag %s for commit %s" % (labelName, commit))
3009         gitStream.write("tag %s\n" % labelName)
3010         gitStream.write("from %s\n" % commit)
3011
3012         if 'Owner' in labelDetails:
3013             owner = labelDetails["Owner"]
3014         else:
3015             owner = None
3016
3017         # Try to use the owner of the p4 label, or failing that,
3018         # the current p4 user id.
3019         if owner:
3020             email = self.make_email(owner)
3021         else:
3022             email = self.make_email(self.p4UserId())
3023         tagger = "%s %s %s" % (email, epoch, self.tz)
3024
3025         gitStream.write("tagger %s\n" % tagger)
3026
3027         print("labelDetails=",labelDetails)
3028         if 'Description' in labelDetails:
3029             description = labelDetails['Description']
3030         else:
3031             description = 'Label from git p4'
3032
3033         gitStream.write("data %d\n" % len(description))
3034         gitStream.write(description)
3035         gitStream.write("\n")
3036
3037     def inClientSpec(self, path):
3038         if not self.clientSpecDirs:
3039             return True
3040         inClientSpec = self.clientSpecDirs.map_in_client(path)
3041         if not inClientSpec and self.verbose:
3042             print('Ignoring file outside of client spec: {0}'.format(path))
3043         return inClientSpec
3044
3045     def hasBranchPrefix(self, path):
3046         if not self.branchPrefixes:
3047             return True
3048         hasPrefix = [p for p in self.branchPrefixes
3049                         if p4PathStartsWith(path, p)]
3050         if not hasPrefix and self.verbose:
3051             print('Ignoring file outside of prefix: {0}'.format(path))
3052         return hasPrefix
3053
3054     def commit(self, details, files, branch, parent = "", allow_empty=False):
3055         epoch = details["time"]
3056         author = details["user"]
3057         jobs = self.extractJobsFromCommit(details)
3058
3059         if self.verbose:
3060             print('commit into {0}'.format(branch))
3061
3062         if self.clientSpecDirs:
3063             self.clientSpecDirs.update_client_spec_path_cache(files)
3064
3065         files = [f for f in files
3066             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3067
3068         if gitConfigBool('git-p4.keepEmptyCommits'):
3069             allow_empty = True
3070
3071         if not files and not allow_empty:
3072             print('Ignoring revision {0} as it would produce an empty commit.'
3073                 .format(details['change']))
3074             return
3075
3076         self.gitStream.write("commit %s\n" % branch)
3077         self.gitStream.write("mark :%s\n" % details["change"])
3078         self.committedChanges.add(int(details["change"]))
3079         committer = ""
3080         if author not in self.users:
3081             self.getUserMapFromPerforceServer()
3082         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3083
3084         self.gitStream.write("committer %s\n" % committer)
3085
3086         self.gitStream.write("data <<EOT\n")
3087         self.gitStream.write(details["desc"])
3088         if len(jobs) > 0:
3089             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3090
3091         if not self.suppress_meta_comment:
3092             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3093                                 (','.join(self.branchPrefixes), details["change"]))
3094             if len(details['options']) > 0:
3095                 self.gitStream.write(": options = %s" % details['options'])
3096             self.gitStream.write("]\n")
3097
3098         self.gitStream.write("EOT\n\n")
3099
3100         if len(parent) > 0:
3101             if self.verbose:
3102                 print("parent %s" % parent)
3103             self.gitStream.write("from %s\n" % parent)
3104
3105         self.streamP4Files(files)
3106         self.gitStream.write("\n")
3107
3108         change = int(details["change"])
3109
3110         if change in self.labels:
3111             label = self.labels[change]
3112             labelDetails = label[0]
3113             labelRevisions = label[1]
3114             if self.verbose:
3115                 print("Change %s is labelled %s" % (change, labelDetails))
3116
3117             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3118                                                 for p in self.branchPrefixes])
3119
3120             if len(files) == len(labelRevisions):
3121
3122                 cleanedFiles = {}
3123                 for info in files:
3124                     if info["action"] in self.delete_actions:
3125                         continue
3126                     cleanedFiles[info["depotFile"]] = info["rev"]
3127
3128                 if cleanedFiles == labelRevisions:
3129                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3130
3131                 else:
3132                     if not self.silent:
3133                         print("Tag %s does not match with change %s: files do not match."
3134                                % (labelDetails["label"], change))
3135
3136             else:
3137                 if not self.silent:
3138                     print("Tag %s does not match with change %s: file count is different."
3139                            % (labelDetails["label"], change))
3140
3141     # Build a dictionary of changelists and labels, for "detect-labels" option.
3142     def getLabels(self):
3143         self.labels = {}
3144
3145         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3146         if len(l) > 0 and not self.silent:
3147             print("Finding files belonging to labels in %s" % self.depotPaths)
3148
3149         for output in l:
3150             label = output["label"]
3151             revisions = {}
3152             newestChange = 0
3153             if self.verbose:
3154                 print("Querying files for label %s" % label)
3155             for file in p4CmdList(["files"] +
3156                                       ["%s...@%s" % (p, label)
3157                                           for p in self.depotPaths]):
3158                 revisions[file["depotFile"]] = file["rev"]
3159                 change = int(file["change"])
3160                 if change > newestChange:
3161                     newestChange = change
3162
3163             self.labels[newestChange] = [output, revisions]
3164
3165         if self.verbose:
3166             print("Label changes: %s" % self.labels.keys())
3167
3168     # Import p4 labels as git tags. A direct mapping does not
3169     # exist, so assume that if all the files are at the same revision
3170     # then we can use that, or it's something more complicated we should
3171     # just ignore.
3172     def importP4Labels(self, stream, p4Labels):
3173         if verbose:
3174             print("import p4 labels: " + ' '.join(p4Labels))
3175
3176         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3177         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3178         if len(validLabelRegexp) == 0:
3179             validLabelRegexp = defaultLabelRegexp
3180         m = re.compile(validLabelRegexp)
3181
3182         for name in p4Labels:
3183             commitFound = False
3184
3185             if not m.match(name):
3186                 if verbose:
3187                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3188                 continue
3189
3190             if name in ignoredP4Labels:
3191                 continue
3192
3193             labelDetails = p4CmdList(['label', "-o", name])[0]
3194
3195             # get the most recent changelist for each file in this label
3196             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3197                                 for p in self.depotPaths])
3198
3199             if 'change' in change:
3200                 # find the corresponding git commit; take the oldest commit
3201                 changelist = int(change['change'])
3202                 if changelist in self.committedChanges:
3203                     gitCommit = ":%d" % changelist       # use a fast-import mark
3204                     commitFound = True
3205                 else:
3206                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3207                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3208                     if len(gitCommit) == 0:
3209                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3210                     else:
3211                         commitFound = True
3212                         gitCommit = gitCommit.strip()
3213
3214                 if commitFound:
3215                     # Convert from p4 time format
3216                     try:
3217                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3218                     except ValueError:
3219                         print("Could not convert label time %s" % labelDetails['Update'])
3220                         tmwhen = 1
3221
3222                     when = int(time.mktime(tmwhen))
3223                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3224                     if verbose:
3225                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3226             else:
3227                 if verbose:
3228                     print("Label %s has no changelists - possibly deleted?" % name)
3229
3230             if not commitFound:
3231                 # We can't import this label; don't try again as it will get very
3232                 # expensive repeatedly fetching all the files for labels that will
3233                 # never be imported. If the label is moved in the future, the
3234                 # ignore will need to be removed manually.
3235                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3236
3237     def guessProjectName(self):
3238         for p in self.depotPaths:
3239             if p.endswith("/"):
3240                 p = p[:-1]
3241             p = p[p.strip().rfind("/") + 1:]
3242             if not p.endswith("/"):
3243                p += "/"
3244             return p
3245
3246     def getBranchMapping(self):
3247         lostAndFoundBranches = set()
3248
3249         user = gitConfig("git-p4.branchUser")
3250         if len(user) > 0:
3251             command = "branches -u %s" % user
3252         else:
3253             command = "branches"
3254
3255         for info in p4CmdList(command):
3256             details = p4Cmd(["branch", "-o", info["branch"]])
3257             viewIdx = 0
3258             while "View%s" % viewIdx in details:
3259                 paths = details["View%s" % viewIdx].split(" ")
3260                 viewIdx = viewIdx + 1
3261                 # require standard //depot/foo/... //depot/bar/... mapping
3262                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3263                     continue
3264                 source = paths[0]
3265                 destination = paths[1]
3266                 ## HACK
3267                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3268                     source = source[len(self.depotPaths[0]):-4]
3269                     destination = destination[len(self.depotPaths[0]):-4]
3270
3271                     if destination in self.knownBranches:
3272                         if not self.silent:
3273                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3274                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3275                         continue
3276
3277                     self.knownBranches[destination] = source
3278
3279                     lostAndFoundBranches.discard(destination)
3280
3281                     if source not in self.knownBranches:
3282                         lostAndFoundBranches.add(source)
3283
3284         # Perforce does not strictly require branches to be defined, so we also
3285         # check git config for a branch list.
3286         #
3287         # Example of branch definition in git config file:
3288         # [git-p4]
3289         #   branchList=main:branchA
3290         #   branchList=main:branchB
3291         #   branchList=branchA:branchC
3292         configBranches = gitConfigList("git-p4.branchList")
3293         for branch in configBranches:
3294             if branch:
3295                 (source, destination) = branch.split(":")
3296                 self.knownBranches[destination] = source
3297
3298                 lostAndFoundBranches.discard(destination)
3299
3300                 if source not in self.knownBranches:
3301                     lostAndFoundBranches.add(source)
3302
3303
3304         for branch in lostAndFoundBranches:
3305             self.knownBranches[branch] = branch
3306
3307     def getBranchMappingFromGitBranches(self):
3308         branches = p4BranchesInGit(self.importIntoRemotes)
3309         for branch in branches.keys():
3310             if branch == "master":
3311                 branch = "main"
3312             else:
3313                 branch = branch[len(self.projectName):]
3314             self.knownBranches[branch] = branch
3315
3316     def updateOptionDict(self, d):
3317         option_keys = {}
3318         if self.keepRepoPath:
3319             option_keys['keepRepoPath'] = 1
3320
3321         d["options"] = ' '.join(sorted(option_keys.keys()))
3322
3323     def readOptions(self, d):
3324         self.keepRepoPath = ('options' in d
3325                              and ('keepRepoPath' in d['options']))
3326
3327     def gitRefForBranch(self, branch):
3328         if branch == "main":
3329             return self.refPrefix + "master"
3330
3331         if len(branch) <= 0:
3332             return branch
3333
3334         return self.refPrefix + self.projectName + branch
3335
3336     def gitCommitByP4Change(self, ref, change):
3337         if self.verbose:
3338             print("looking in ref " + ref + " for change %s using bisect..." % change)
3339
3340         earliestCommit = ""
3341         latestCommit = parseRevision(ref)
3342
3343         while True:
3344             if self.verbose:
3345                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3346             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3347             if len(next) == 0:
3348                 if self.verbose:
3349                     print("argh")
3350                 return ""
3351             log = extractLogMessageFromGitCommit(next)
3352             settings = extractSettingsGitLog(log)
3353             currentChange = int(settings['change'])
3354             if self.verbose:
3355                 print("current change %s" % currentChange)
3356
3357             if currentChange == change:
3358                 if self.verbose:
3359                     print("found %s" % next)
3360                 return next
3361
3362             if currentChange < change:
3363                 earliestCommit = "^%s" % next
3364             else:
3365                 if next == latestCommit:
3366                     die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3367                 latestCommit = "%s^@" % next
3368
3369         return ""
3370
3371     def importNewBranch(self, branch, maxChange):
3372         # make fast-import flush all changes to disk and update the refs using the checkpoint
3373         # command so that we can try to find the branch parent in the git history
3374         self.gitStream.write("checkpoint\n\n");
3375         self.gitStream.flush();
3376         branchPrefix = self.depotPaths[0] + branch + "/"
3377         range = "@1,%s" % maxChange
3378         #print "prefix" + branchPrefix
3379         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3380         if len(changes) <= 0:
3381             return False
3382         firstChange = changes[0]
3383         #print "first change in branch: %s" % firstChange
3384         sourceBranch = self.knownBranches[branch]
3385         sourceDepotPath = self.depotPaths[0] + sourceBranch
3386         sourceRef = self.gitRefForBranch(sourceBranch)
3387         #print "source " + sourceBranch
3388
3389         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3390         #print "branch parent: %s" % branchParentChange
3391         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3392         if len(gitParent) > 0:
3393             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3394             #print "parent git commit: %s" % gitParent
3395
3396         self.importChanges(changes)
3397         return True
3398
3399     def searchParent(self, parent, branch, target):
3400         parentFound = False
3401         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3402                                      "--no-merges", parent]):
3403             blob = blob.strip()
3404             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3405                 parentFound = True
3406                 if self.verbose:
3407                     print("Found parent of %s in commit %s" % (branch, blob))
3408                 break
3409         if parentFound:
3410             return blob
3411         else:
3412             return None
3413
3414     def importChanges(self, changes, origin_revision=0):
3415         cnt = 1
3416         for change in changes:
3417             description = p4_describe(change)
3418             self.updateOptionDict(description)
3419
3420             if not self.silent:
3421                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3422                 sys.stdout.flush()
3423             cnt = cnt + 1
3424
3425             try:
3426                 if self.detectBranches:
3427                     branches = self.splitFilesIntoBranches(description)
3428                     for branch in branches.keys():
3429                         ## HACK  --hwn
3430                         branchPrefix = self.depotPaths[0] + branch + "/"
3431                         self.branchPrefixes = [ branchPrefix ]
3432
3433                         parent = ""
3434
3435                         filesForCommit = branches[branch]
3436
3437                         if self.verbose:
3438                             print("branch is %s" % branch)
3439
3440                         self.updatedBranches.add(branch)
3441
3442                         if branch not in self.createdBranches:
3443                             self.createdBranches.add(branch)
3444                             parent = self.knownBranches[branch]
3445                             if parent == branch:
3446                                 parent = ""
3447                             else:
3448                                 fullBranch = self.projectName + branch
3449                                 if fullBranch not in self.p4BranchesInGit:
3450                                     if not self.silent:
3451                                         print("\n    Importing new branch %s" % fullBranch);
3452                                     if self.importNewBranch(branch, change - 1):
3453                                         parent = ""
3454                                         self.p4BranchesInGit.append(fullBranch)
3455                                     if not self.silent:
3456                                         print("\n    Resuming with change %s" % change);
3457
3458                                 if self.verbose:
3459                                     print("parent determined through known branches: %s" % parent)
3460
3461                         branch = self.gitRefForBranch(branch)
3462                         parent = self.gitRefForBranch(parent)
3463
3464                         if self.verbose:
3465                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3466
3467                         if len(parent) == 0 and branch in self.initialParents:
3468                             parent = self.initialParents[branch]
3469                             del self.initialParents[branch]
3470
3471                         blob = None
3472                         if len(parent) > 0:
3473                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3474                             if self.verbose:
3475                                 print("Creating temporary branch: " + tempBranch)
3476                             self.commit(description, filesForCommit, tempBranch)
3477                             self.tempBranches.append(tempBranch)
3478                             self.checkpoint()
3479                             blob = self.searchParent(parent, branch, tempBranch)
3480                         if blob:
3481                             self.commit(description, filesForCommit, branch, blob)
3482                         else:
3483                             if self.verbose:
3484                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3485                             self.commit(description, filesForCommit, branch, parent)
3486                 else:
3487                     files = self.extractFilesFromCommit(description)
3488                     self.commit(description, files, self.branch,
3489                                 self.initialParent)
3490                     # only needed once, to connect to the previous commit
3491                     self.initialParent = ""
3492             except IOError:
3493                 print(self.gitError.read())
3494                 sys.exit(1)
3495
3496     def sync_origin_only(self):
3497         if self.syncWithOrigin:
3498             self.hasOrigin = originP4BranchesExist()
3499             if self.hasOrigin:
3500                 if not self.silent:
3501                     print('Syncing with origin first, using "git fetch origin"')
3502                 system("git fetch origin")
3503
3504     def importHeadRevision(self, revision):
3505         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3506
3507         details = {}
3508         details["user"] = "git perforce import user"
3509         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3510                            % (' '.join(self.depotPaths), revision))
3511         details["change"] = revision
3512         newestRevision = 0
3513
3514         fileCnt = 0
3515         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3516
3517         for info in p4CmdList(["files"] + fileArgs):
3518
3519             if 'code' in info and info['code'] == 'error':
3520                 sys.stderr.write("p4 returned an error: %s\n"
3521                                  % info['data'])
3522                 if info['data'].find("must refer to client") >= 0:
3523                     sys.stderr.write("This particular p4 error is misleading.\n")
3524                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3525                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3526                 sys.exit(1)
3527             if 'p4ExitCode' in info:
3528                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3529                 sys.exit(1)
3530
3531
3532             change = int(info["change"])
3533             if change > newestRevision:
3534                 newestRevision = change
3535
3536             if info["action"] in self.delete_actions:
3537                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3538                 #fileCnt = fileCnt + 1
3539                 continue
3540
3541             for prop in ["depotFile", "rev", "action", "type" ]:
3542                 details["%s%s" % (prop, fileCnt)] = info[prop]
3543
3544             fileCnt = fileCnt + 1
3545
3546         details["change"] = newestRevision
3547
3548         # Use time from top-most change so that all git p4 clones of
3549         # the same p4 repo have the same commit SHA1s.
3550         res = p4_describe(newestRevision)
3551         details["time"] = res["time"]
3552
3553         self.updateOptionDict(details)
3554         try:
3555             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3556         except IOError as err:
3557             print("IO error with git fast-import. Is your git version recent enough?")
3558             print("IO error details: {}".format(err))
3559             print(self.gitError.read())
3560
3561
3562     def importRevisions(self, args, branch_arg_given):
3563         changes = []
3564
3565         if len(self.changesFile) > 0:
3566             with open(self.changesFile) as f:
3567                 output = f.readlines()
3568             changeSet = set()
3569             for line in output:
3570                 changeSet.add(int(line))
3571
3572             for change in changeSet:
3573                 changes.append(change)
3574
3575             changes.sort()
3576         else:
3577             # catch "git p4 sync" with no new branches, in a repo that
3578             # does not have any existing p4 branches
3579             if len(args) == 0:
3580                 if not self.p4BranchesInGit:
3581                     raise P4CommandException("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3582
3583                 # The default branch is master, unless --branch is used to
3584                 # specify something else.  Make sure it exists, or complain
3585                 # nicely about how to use --branch.
3586                 if not self.detectBranches:
3587                     if not branch_exists(self.branch):
3588                         if branch_arg_given:
3589                             raise P4CommandException("Error: branch %s does not exist." % self.branch)
3590                         else:
3591                             raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3592                                 self.branch)
3593
3594             if self.verbose:
3595                 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3596                                                           self.changeRange))
3597             changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3598
3599             if len(self.maxChanges) > 0:
3600                 changes = changes[:min(int(self.maxChanges), len(changes))]
3601
3602         if len(changes) == 0:
3603             if not self.silent:
3604                 print("No changes to import!")
3605         else:
3606             if not self.silent and not self.detectBranches:
3607                 print("Import destination: %s" % self.branch)
3608
3609             self.updatedBranches = set()
3610
3611             if not self.detectBranches:
3612                 if args:
3613                     # start a new branch
3614                     self.initialParent = ""
3615                 else:
3616                     # build on a previous revision
3617                     self.initialParent = parseRevision(self.branch)
3618
3619             self.importChanges(changes)
3620
3621             if not self.silent:
3622                 print("")
3623                 if len(self.updatedBranches) > 0:
3624                     sys.stdout.write("Updated branches: ")
3625                     for b in self.updatedBranches:
3626                         sys.stdout.write("%s " % b)
3627                     sys.stdout.write("\n")
3628
3629     def openStreams(self):
3630         self.importProcess = subprocess.Popen(["git", "fast-import"],
3631                                               stdin=subprocess.PIPE,
3632                                               stdout=subprocess.PIPE,
3633                                               stderr=subprocess.PIPE);
3634         self.gitOutput = self.importProcess.stdout
3635         self.gitStream = self.importProcess.stdin
3636         self.gitError = self.importProcess.stderr
3637
3638     def closeStreams(self):
3639         if self.gitStream is None:
3640             return
3641         self.gitStream.close()
3642         if self.importProcess.wait() != 0:
3643             die("fast-import failed: %s" % self.gitError.read())
3644         self.gitOutput.close()
3645         self.gitError.close()
3646         self.gitStream = None
3647
3648     def run(self, args):
3649         if self.importIntoRemotes:
3650             self.refPrefix = "refs/remotes/p4/"
3651         else:
3652             self.refPrefix = "refs/heads/p4/"
3653
3654         self.sync_origin_only()
3655
3656         branch_arg_given = bool(self.branch)
3657         if len(self.branch) == 0:
3658             self.branch = self.refPrefix + "master"
3659             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3660                 system("git update-ref %s refs/heads/p4" % self.branch)
3661                 system("git branch -D p4")
3662
3663         # accept either the command-line option, or the configuration variable
3664         if self.useClientSpec:
3665             # will use this after clone to set the variable
3666             self.useClientSpec_from_options = True
3667         else:
3668             if gitConfigBool("git-p4.useclientspec"):
3669                 self.useClientSpec = True
3670         if self.useClientSpec:
3671             self.clientSpecDirs = getClientSpec()
3672
3673         # TODO: should always look at previous commits,
3674         # merge with previous imports, if possible.
3675         if args == []:
3676             if self.hasOrigin:
3677                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3678
3679             # branches holds mapping from branch name to sha1
3680             branches = p4BranchesInGit(self.importIntoRemotes)
3681
3682             # restrict to just this one, disabling detect-branches
3683             if branch_arg_given:
3684                 short = self.branch.split("/")[-1]
3685                 if short in branches:
3686                     self.p4BranchesInGit = [ short ]
3687             else:
3688                 self.p4BranchesInGit = branches.keys()
3689
3690             if len(self.p4BranchesInGit) > 1:
3691                 if not self.silent:
3692                     print("Importing from/into multiple branches")
3693                 self.detectBranches = True
3694                 for branch in branches.keys():
3695                     self.initialParents[self.refPrefix + branch] = \
3696                         branches[branch]
3697
3698             if self.verbose:
3699                 print("branches: %s" % self.p4BranchesInGit)
3700
3701             p4Change = 0
3702             for branch in self.p4BranchesInGit:
3703                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3704
3705                 settings = extractSettingsGitLog(logMsg)
3706
3707                 self.readOptions(settings)
3708                 if ('depot-paths' in settings
3709                     and 'change' in settings):
3710                     change = int(settings['change']) + 1
3711                     p4Change = max(p4Change, change)
3712
3713                     depotPaths = sorted(settings['depot-paths'])
3714                     if self.previousDepotPaths == []:
3715                         self.previousDepotPaths = depotPaths
3716                     else:
3717                         paths = []
3718                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3719                             prev_list = prev.split("/")
3720                             cur_list = cur.split("/")
3721                             for i in range(0, min(len(cur_list), len(prev_list))):
3722                                 if cur_list[i] != prev_list[i]:
3723                                     i = i - 1
3724                                     break
3725
3726                             paths.append ("/".join(cur_list[:i + 1]))
3727
3728                         self.previousDepotPaths = paths
3729
3730             if p4Change > 0:
3731                 self.depotPaths = sorted(self.previousDepotPaths)
3732                 self.changeRange = "@%s,#head" % p4Change
3733                 if not self.silent and not self.detectBranches:
3734                     print("Performing incremental import into %s git branch" % self.branch)
3735
3736         # accept multiple ref name abbreviations:
3737         #    refs/foo/bar/branch -> use it exactly
3738         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3739         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3740         if not self.branch.startswith("refs/"):
3741             if self.importIntoRemotes:
3742                 prepend = "refs/remotes/"
3743             else:
3744                 prepend = "refs/heads/"
3745             if not self.branch.startswith("p4/"):
3746                 prepend += "p4/"
3747             self.branch = prepend + self.branch
3748
3749         if len(args) == 0 and self.depotPaths:
3750             if not self.silent:
3751                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3752         else:
3753             if self.depotPaths and self.depotPaths != args:
3754                 print("previous import used depot path %s and now %s was specified. "
3755                        "This doesn't work!" % (' '.join (self.depotPaths),
3756                                                ' '.join (args)))
3757                 sys.exit(1)
3758
3759             self.depotPaths = sorted(args)
3760
3761         revision = ""
3762         self.users = {}
3763
3764         # Make sure no revision specifiers are used when --changesfile
3765         # is specified.
3766         bad_changesfile = False
3767         if len(self.changesFile) > 0:
3768             for p in self.depotPaths:
3769                 if p.find("@") >= 0 or p.find("#") >= 0:
3770                     bad_changesfile = True
3771                     break
3772         if bad_changesfile:
3773             die("Option --changesfile is incompatible with revision specifiers")
3774
3775         newPaths = []
3776         for p in self.depotPaths:
3777             if p.find("@") != -1:
3778                 atIdx = p.index("@")
3779                 self.changeRange = p[atIdx:]
3780                 if self.changeRange == "@all":
3781                     self.changeRange = ""
3782                 elif ',' not in self.changeRange:
3783                     revision = self.changeRange
3784                     self.changeRange = ""
3785                 p = p[:atIdx]
3786             elif p.find("#") != -1:
3787                 hashIdx = p.index("#")
3788                 revision = p[hashIdx:]
3789                 p = p[:hashIdx]
3790             elif self.previousDepotPaths == []:
3791                 # pay attention to changesfile, if given, else import
3792                 # the entire p4 tree at the head revision
3793                 if len(self.changesFile) == 0:
3794                     revision = "#head"
3795
3796             p = re.sub ("\.\.\.$", "", p)
3797             if not p.endswith("/"):
3798                 p += "/"
3799
3800             newPaths.append(p)
3801
3802         self.depotPaths = newPaths
3803
3804         # --detect-branches may change this for each branch
3805         self.branchPrefixes = self.depotPaths
3806
3807         self.loadUserMapFromCache()
3808         self.labels = {}
3809         if self.detectLabels:
3810             self.getLabels();
3811
3812         if self.detectBranches:
3813             ## FIXME - what's a P4 projectName ?
3814             self.projectName = self.guessProjectName()
3815
3816             if self.hasOrigin:
3817                 self.getBranchMappingFromGitBranches()
3818             else:
3819                 self.getBranchMapping()
3820             if self.verbose:
3821                 print("p4-git branches: %s" % self.p4BranchesInGit)
3822                 print("initial parents: %s" % self.initialParents)
3823             for b in self.p4BranchesInGit:
3824                 if b != "master":
3825
3826                     ## FIXME
3827                     b = b[len(self.projectName):]
3828                 self.createdBranches.add(b)
3829
3830         p4_check_access()
3831
3832         self.openStreams()
3833
3834         err = None
3835
3836         try:
3837             if revision:
3838                 self.importHeadRevision(revision)
3839             else:
3840                 self.importRevisions(args, branch_arg_given)
3841
3842             if gitConfigBool("git-p4.importLabels"):
3843                 self.importLabels = True
3844
3845             if self.importLabels:
3846                 p4Labels = getP4Labels(self.depotPaths)
3847                 gitTags = getGitTags()
3848
3849                 missingP4Labels = p4Labels - gitTags
3850                 self.importP4Labels(self.gitStream, missingP4Labels)
3851
3852         except P4CommandException as e:
3853             err = e
3854
3855         finally:
3856             self.closeStreams()
3857
3858         if err:
3859             die(str(err))
3860
3861         # Cleanup temporary branches created during import
3862         if self.tempBranches != []:
3863             for branch in self.tempBranches:
3864                 read_pipe("git update-ref -d %s" % branch)
3865             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3866
3867         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3868         # a convenient shortcut refname "p4".
3869         if self.importIntoRemotes:
3870             head_ref = self.refPrefix + "HEAD"
3871             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3872                 system(["git", "symbolic-ref", head_ref, self.branch])
3873
3874         return True
3875
3876 class P4Rebase(Command):
3877     def __init__(self):
3878         Command.__init__(self)
3879         self.options = [
3880                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3881         ]
3882         self.importLabels = False
3883         self.description = ("Fetches the latest revision from perforce and "
3884                             + "rebases the current work (branch) against it")
3885
3886     def run(self, args):
3887         sync = P4Sync()
3888         sync.importLabels = self.importLabels
3889         sync.run([])
3890
3891         return self.rebase()
3892
3893     def rebase(self):
3894         if os.system("git update-index --refresh") != 0:
3895             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3896         if len(read_pipe("git diff-index HEAD --")) > 0:
3897             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3898
3899         [upstream, settings] = findUpstreamBranchPoint()
3900         if len(upstream) == 0:
3901             die("Cannot find upstream branchpoint for rebase")
3902
3903         # the branchpoint may be p4/foo~3, so strip off the parent
3904         upstream = re.sub("~[0-9]+$", "", upstream)
3905
3906         print("Rebasing the current branch onto %s" % upstream)
3907         oldHead = read_pipe("git rev-parse HEAD").strip()
3908         system("git rebase %s" % upstream)
3909         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3910         return True
3911
3912 class P4Clone(P4Sync):
3913     def __init__(self):
3914         P4Sync.__init__(self)
3915         self.description = "Creates a new git repository and imports from Perforce into it"
3916         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3917         self.options += [
3918             optparse.make_option("--destination", dest="cloneDestination",
3919                                  action='store', default=None,
3920                                  help="where to leave result of the clone"),
3921             optparse.make_option("--bare", dest="cloneBare",
3922                                  action="store_true", default=False),
3923         ]
3924         self.cloneDestination = None
3925         self.needsGit = False
3926         self.cloneBare = False
3927
3928     def defaultDestination(self, args):
3929         ## TODO: use common prefix of args?
3930         depotPath = args[0]
3931         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3932         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3933         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3934         depotDir = re.sub(r"/$", "", depotDir)
3935         return os.path.split(depotDir)[1]
3936
3937     def run(self, args):
3938         if len(args) < 1:
3939             return False
3940
3941         if self.keepRepoPath and not self.cloneDestination:
3942             sys.stderr.write("Must specify destination for --keep-path\n")
3943             sys.exit(1)
3944
3945         depotPaths = args
3946
3947         if not self.cloneDestination and len(depotPaths) > 1:
3948             self.cloneDestination = depotPaths[-1]
3949             depotPaths = depotPaths[:-1]
3950
3951         for p in depotPaths:
3952             if not p.startswith("//"):
3953                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3954                 return False
3955
3956         if not self.cloneDestination:
3957             self.cloneDestination = self.defaultDestination(args)
3958
3959         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3960
3961         if not os.path.exists(self.cloneDestination):
3962             os.makedirs(self.cloneDestination)
3963         chdir(self.cloneDestination)
3964
3965         init_cmd = [ "git", "init" ]
3966         if self.cloneBare:
3967             init_cmd.append("--bare")
3968         retcode = subprocess.call(init_cmd)
3969         if retcode:
3970             raise CalledProcessError(retcode, init_cmd)
3971
3972         if not P4Sync.run(self, depotPaths):
3973             return False
3974
3975         # create a master branch and check out a work tree
3976         if gitBranchExists(self.branch):
3977             system([ "git", "branch", "master", self.branch ])
3978             if not self.cloneBare:
3979                 system([ "git", "checkout", "-f" ])
3980         else:
3981             print('Not checking out any branch, use ' \
3982                   '"git checkout -q -b master <branch>"')
3983
3984         # auto-set this variable if invoked with --use-client-spec
3985         if self.useClientSpec_from_options:
3986             system("git config --bool git-p4.useclientspec true")
3987
3988         return True
3989
3990 class P4Unshelve(Command):
3991     def __init__(self):
3992         Command.__init__(self)
3993         self.options = []
3994         self.origin = "HEAD"
3995         self.description = "Unshelve a P4 changelist into a git commit"
3996         self.usage = "usage: %prog [options] changelist"
3997         self.options += [
3998                 optparse.make_option("--origin", dest="origin",
3999                     help="Use this base revision instead of the default (%s)" % self.origin),
4000         ]
4001         self.verbose = False
4002         self.noCommit = False
4003         self.destbranch = "refs/remotes/p4-unshelved"
4004
4005     def renameBranch(self, branch_name):
4006         """ Rename the existing branch to branch_name.N
4007         """
4008
4009         found = True
4010         for i in range(0,1000):
4011             backup_branch_name = "{0}.{1}".format(branch_name, i)
4012             if not gitBranchExists(backup_branch_name):
4013                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
4014                 gitDeleteRef(branch_name)
4015                 found = True
4016                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
4017                 break
4018
4019         if not found:
4020             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
4021
4022     def findLastP4Revision(self, starting_point):
4023         """ Look back from starting_point for the first commit created by git-p4
4024             to find the P4 commit we are based on, and the depot-paths.
4025         """
4026
4027         for parent in (range(65535)):
4028             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
4029             settings = extractSettingsGitLog(log)
4030             if 'change' in settings:
4031                 return settings
4032
4033         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4034
4035     def createShelveParent(self, change, branch_name, sync, origin):
4036         """ Create a commit matching the parent of the shelved changelist 'change'
4037         """
4038         parent_description = p4_describe(change, shelved=True)
4039         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4040         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4041
4042         parent_files = []
4043         for f in files:
4044             # if it was added in the shelved changelist, it won't exist in the parent
4045             if f['action'] in self.add_actions:
4046                 continue
4047
4048             # if it was deleted in the shelved changelist it must not be deleted
4049             # in the parent - we might even need to create it if the origin branch
4050             # does not have it
4051             if f['action'] in self.delete_actions:
4052                 f['action'] = 'add'
4053
4054             parent_files.append(f)
4055
4056         sync.commit(parent_description, parent_files, branch_name,
4057                 parent=origin, allow_empty=True)
4058         print("created parent commit for {0} based on {1} in {2}".format(
4059             change, self.origin, branch_name))
4060
4061     def run(self, args):
4062         if len(args) != 1:
4063             return False
4064
4065         if not gitBranchExists(self.origin):
4066             sys.exit("origin branch {0} does not exist".format(self.origin))
4067
4068         sync = P4Sync()
4069         changes = args
4070
4071         # only one change at a time
4072         change = changes[0]
4073
4074         # if the target branch already exists, rename it
4075         branch_name = "{0}/{1}".format(self.destbranch, change)
4076         if gitBranchExists(branch_name):
4077             self.renameBranch(branch_name)
4078         sync.branch = branch_name
4079
4080         sync.verbose = self.verbose
4081         sync.suppress_meta_comment = True
4082
4083         settings = self.findLastP4Revision(self.origin)
4084         sync.depotPaths = settings['depot-paths']
4085         sync.branchPrefixes = sync.depotPaths
4086
4087         sync.openStreams()
4088         sync.loadUserMapFromCache()
4089         sync.silent = True
4090
4091         # create a commit for the parent of the shelved changelist
4092         self.createShelveParent(change, branch_name, sync, self.origin)
4093
4094         # create the commit for the shelved changelist itself
4095         description = p4_describe(change, True)
4096         files = sync.extractFilesFromCommit(description, True, change)
4097
4098         sync.commit(description, files, branch_name, "")
4099         sync.closeStreams()
4100
4101         print("unshelved changelist {0} into {1}".format(change, branch_name))
4102
4103         return True
4104
4105 class P4Branches(Command):
4106     def __init__(self):
4107         Command.__init__(self)
4108         self.options = [ ]
4109         self.description = ("Shows the git branches that hold imports and their "
4110                             + "corresponding perforce depot paths")
4111         self.verbose = False
4112
4113     def run(self, args):
4114         if originP4BranchesExist():
4115             createOrUpdateBranchesFromOrigin()
4116
4117         cmdline = "git rev-parse --symbolic "
4118         cmdline += " --remotes"
4119
4120         for line in read_pipe_lines(cmdline):
4121             line = line.strip()
4122
4123             if not line.startswith('p4/') or line == "p4/HEAD":
4124                 continue
4125             branch = line
4126
4127             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4128             settings = extractSettingsGitLog(log)
4129
4130             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4131         return True
4132
4133 class HelpFormatter(optparse.IndentedHelpFormatter):
4134     def __init__(self):
4135         optparse.IndentedHelpFormatter.__init__(self)
4136
4137     def format_description(self, description):
4138         if description:
4139             return description + "\n"
4140         else:
4141             return ""
4142
4143 def printUsage(commands):
4144     print("usage: %s <command> [options]" % sys.argv[0])
4145     print("")
4146     print("valid commands: %s" % ", ".join(commands))
4147     print("")
4148     print("Try %s <command> --help for command specific help." % sys.argv[0])
4149     print("")
4150
4151 commands = {
4152     "debug" : P4Debug,
4153     "submit" : P4Submit,
4154     "commit" : P4Submit,
4155     "sync" : P4Sync,
4156     "rebase" : P4Rebase,
4157     "clone" : P4Clone,
4158     "rollback" : P4RollBack,
4159     "branches" : P4Branches,
4160     "unshelve" : P4Unshelve,
4161 }
4162
4163
4164 def main():
4165     if len(sys.argv[1:]) == 0:
4166         printUsage(commands.keys())
4167         sys.exit(2)
4168
4169     cmdName = sys.argv[1]
4170     try:
4171         klass = commands[cmdName]
4172         cmd = klass()
4173     except KeyError:
4174         print("unknown command %s" % cmdName)
4175         print("")
4176         printUsage(commands.keys())
4177         sys.exit(2)
4178
4179     options = cmd.options
4180     cmd.gitdir = os.environ.get("GIT_DIR", None)
4181
4182     args = sys.argv[2:]
4183
4184     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4185     if cmd.needsGit:
4186         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4187
4188     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4189                                    options,
4190                                    description = cmd.description,
4191                                    formatter = HelpFormatter())
4192
4193     try:
4194         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4195     except:
4196         parser.print_help()
4197         raise
4198
4199     global verbose
4200     verbose = cmd.verbose
4201     if cmd.needsGit:
4202         if cmd.gitdir == None:
4203             cmd.gitdir = os.path.abspath(".git")
4204             if not isValidGitDir(cmd.gitdir):
4205                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4206                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4207                 if os.path.exists(cmd.gitdir):
4208                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4209                     if len(cdup) > 0:
4210                         chdir(cdup);
4211
4212         if not isValidGitDir(cmd.gitdir):
4213             if isValidGitDir(cmd.gitdir + "/.git"):
4214                 cmd.gitdir += "/.git"
4215             else:
4216                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4217
4218         # so git commands invoked from the P4 workspace will succeed
4219         os.environ["GIT_DIR"] = cmd.gitdir
4220
4221     if not cmd.run(args):
4222         parser.print_help()
4223         sys.exit(2)
4224
4225
4226 if __name__ == '__main__':
4227     main()