Merge branch 'bc/sha-256-part-1-of-4'
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
17 #
18 import sys
19 if sys.version_info.major < 3 and sys.version_info.minor < 7:
20     sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
21     sys.exit(1)
22 import os
23 import optparse
24 import functools
25 import marshal
26 import subprocess
27 import tempfile
28 import time
29 import platform
30 import re
31 import shutil
32 import stat
33 import zipfile
34 import zlib
35 import ctypes
36 import errno
37
38 # On python2.7 where raw_input() and input() are both availble,
39 # we want raw_input's semantics, but aliased to input for python3
40 # compatibility
41 # support basestring in python3
42 try:
43     if raw_input and input:
44         input = raw_input
45 except:
46     pass
47
48 verbose = False
49
50 # Only labels/tags matching this will be imported/exported
51 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
52
53 # The block size is reduced automatically if required
54 defaultBlockSize = 1<<20
55
56 p4_access_checked = False
57
58 def p4_build_cmd(cmd):
59     """Build a suitable p4 command line.
60
61     This consolidates building and returning a p4 command line into one
62     location. It means that hooking into the environment, or other configuration
63     can be done more easily.
64     """
65     real_cmd = ["p4"]
66
67     user = gitConfig("git-p4.user")
68     if len(user) > 0:
69         real_cmd += ["-u",user]
70
71     password = gitConfig("git-p4.password")
72     if len(password) > 0:
73         real_cmd += ["-P", password]
74
75     port = gitConfig("git-p4.port")
76     if len(port) > 0:
77         real_cmd += ["-p", port]
78
79     host = gitConfig("git-p4.host")
80     if len(host) > 0:
81         real_cmd += ["-H", host]
82
83     client = gitConfig("git-p4.client")
84     if len(client) > 0:
85         real_cmd += ["-c", client]
86
87     retries = gitConfigInt("git-p4.retries")
88     if retries is None:
89         # Perform 3 retries by default
90         retries = 3
91     if retries > 0:
92         # Provide a way to not pass this option by setting git-p4.retries to 0
93         real_cmd += ["-r", str(retries)]
94
95     if not isinstance(cmd, list):
96         real_cmd = ' '.join(real_cmd) + ' ' + cmd
97     else:
98         real_cmd += cmd
99
100     # now check that we can actually talk to the server
101     global p4_access_checked
102     if not p4_access_checked:
103         p4_access_checked = True    # suppress access checks in p4_check_access itself
104         p4_check_access()
105
106     return real_cmd
107
108 def git_dir(path):
109     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
110         This won't automatically add ".git" to a directory.
111     """
112     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
113     if not d or len(d) == 0:
114         return None
115     else:
116         return d
117
118 def chdir(path, is_client_path=False):
119     """Do chdir to the given path, and set the PWD environment
120        variable for use by P4.  It does not look at getcwd() output.
121        Since we're not using the shell, it is necessary to set the
122        PWD environment variable explicitly.
123
124        Normally, expand the path to force it to be absolute.  This
125        addresses the use of relative path names inside P4 settings,
126        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
127        as given; it looks for .p4config using PWD.
128
129        If is_client_path, the path was handed to us directly by p4,
130        and may be a symbolic link.  Do not call os.getcwd() in this
131        case, because it will cause p4 to think that PWD is not inside
132        the client path.
133        """
134
135     os.chdir(path)
136     if not is_client_path:
137         path = os.getcwd()
138     os.environ['PWD'] = path
139
140 def calcDiskFree():
141     """Return free space in bytes on the disk of the given dirname."""
142     if platform.system() == 'Windows':
143         free_bytes = ctypes.c_ulonglong(0)
144         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
145         return free_bytes.value
146     else:
147         st = os.statvfs(os.getcwd())
148         return st.f_bavail * st.f_frsize
149
150 def die(msg):
151     """ Terminate execution. Make sure that any running child processes have been wait()ed for before
152         calling this.
153     """
154     if verbose:
155         raise Exception(msg)
156     else:
157         sys.stderr.write(msg + "\n")
158         sys.exit(1)
159
160 def prompt(prompt_text):
161     """ Prompt the user to choose one of the choices
162
163     Choices are identified in the prompt_text by square brackets around
164     a single letter option.
165     """
166     choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
167     while True:
168         response = input(prompt_text).strip().lower()
169         if not response:
170             continue
171         response = response[0]
172         if response in choices:
173             return response
174
175 # We need different encoding/decoding strategies for text data being passed
176 # around in pipes depending on python version
177 if bytes is not str:
178     # For python3, always encode and decode as appropriate
179     def decode_text_stream(s):
180         return s.decode() if isinstance(s, bytes) else s
181     def encode_text_stream(s):
182         return s.encode() if isinstance(s, str) else s
183 else:
184     # For python2.7, pass read strings as-is, but also allow writing unicode
185     def decode_text_stream(s):
186         return s
187     def encode_text_stream(s):
188         return s.encode('utf_8') if isinstance(s, unicode) else s
189
190 def decode_path(path):
191     """Decode a given string (bytes or otherwise) using configured path encoding options
192     """
193     encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
194     if bytes is not str:
195         return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
196     else:
197         try:
198             path.decode('ascii')
199         except:
200             path = path.decode(encoding, errors='replace')
201             if verbose:
202                 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
203         return path
204
205 def write_pipe(c, stdin):
206     if verbose:
207         sys.stderr.write('Writing pipe: %s\n' % str(c))
208
209     expand = not isinstance(c, list)
210     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
211     pipe = p.stdin
212     val = pipe.write(stdin)
213     pipe.close()
214     if p.wait():
215         die('Command failed: %s' % str(c))
216
217     return val
218
219 def p4_write_pipe(c, stdin):
220     real_cmd = p4_build_cmd(c)
221     if bytes is not str and isinstance(stdin, str):
222         stdin = encode_text_stream(stdin)
223     return write_pipe(real_cmd, stdin)
224
225 def read_pipe_full(c):
226     """ Read output from  command. Returns a tuple
227         of the return status, stdout text and stderr
228         text.
229     """
230     if verbose:
231         sys.stderr.write('Reading pipe: %s\n' % str(c))
232
233     expand = not isinstance(c, list)
234     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
235     (out, err) = p.communicate()
236     return (p.returncode, out, decode_text_stream(err))
237
238 def read_pipe(c, ignore_error=False, raw=False):
239     """ Read output from  command. Returns the output text on
240         success. On failure, terminates execution, unless
241         ignore_error is True, when it returns an empty string.
242
243         If raw is True, do not attempt to decode output text.
244     """
245     (retcode, out, err) = read_pipe_full(c)
246     if retcode != 0:
247         if ignore_error:
248             out = ""
249         else:
250             die('Command failed: %s\nError: %s' % (str(c), err))
251     if not raw:
252         out = decode_text_stream(out)
253     return out
254
255 def read_pipe_text(c):
256     """ Read output from a command with trailing whitespace stripped.
257         On error, returns None.
258     """
259     (retcode, out, err) = read_pipe_full(c)
260     if retcode != 0:
261         return None
262     else:
263         return decode_text_stream(out).rstrip()
264
265 def p4_read_pipe(c, ignore_error=False, raw=False):
266     real_cmd = p4_build_cmd(c)
267     return read_pipe(real_cmd, ignore_error, raw=raw)
268
269 def read_pipe_lines(c):
270     if verbose:
271         sys.stderr.write('Reading pipe: %s\n' % str(c))
272
273     expand = not isinstance(c, list)
274     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
275     pipe = p.stdout
276     val = [decode_text_stream(line) for line in pipe.readlines()]
277     if pipe.close() or p.wait():
278         die('Command failed: %s' % str(c))
279     return val
280
281 def p4_read_pipe_lines(c):
282     """Specifically invoke p4 on the command supplied. """
283     real_cmd = p4_build_cmd(c)
284     return read_pipe_lines(real_cmd)
285
286 def p4_has_command(cmd):
287     """Ask p4 for help on this command.  If it returns an error, the
288        command does not exist in this version of p4."""
289     real_cmd = p4_build_cmd(["help", cmd])
290     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
291                                    stderr=subprocess.PIPE)
292     p.communicate()
293     return p.returncode == 0
294
295 def p4_has_move_command():
296     """See if the move command exists, that it supports -k, and that
297        it has not been administratively disabled.  The arguments
298        must be correct, but the filenames do not have to exist.  Use
299        ones with wildcards so even if they exist, it will fail."""
300
301     if not p4_has_command("move"):
302         return False
303     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
304     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
305     (out, err) = p.communicate()
306     err = decode_text_stream(err)
307     # return code will be 1 in either case
308     if err.find("Invalid option") >= 0:
309         return False
310     if err.find("disabled") >= 0:
311         return False
312     # assume it failed because @... was invalid changelist
313     return True
314
315 def system(cmd, ignore_error=False):
316     expand = not isinstance(cmd, list)
317     if verbose:
318         sys.stderr.write("executing %s\n" % str(cmd))
319     retcode = subprocess.call(cmd, shell=expand)
320     if retcode and not ignore_error:
321         raise CalledProcessError(retcode, cmd)
322
323     return retcode
324
325 def p4_system(cmd):
326     """Specifically invoke p4 as the system command. """
327     real_cmd = p4_build_cmd(cmd)
328     expand = not isinstance(real_cmd, list)
329     retcode = subprocess.call(real_cmd, shell=expand)
330     if retcode:
331         raise CalledProcessError(retcode, real_cmd)
332
333 def die_bad_access(s):
334     die("failure accessing depot: {0}".format(s.rstrip()))
335
336 def p4_check_access(min_expiration=1):
337     """ Check if we can access Perforce - account still logged in
338     """
339     results = p4CmdList(["login", "-s"])
340
341     if len(results) == 0:
342         # should never get here: always get either some results, or a p4ExitCode
343         assert("could not parse response from perforce")
344
345     result = results[0]
346
347     if 'p4ExitCode' in result:
348         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
349         die_bad_access("could not run p4")
350
351     code = result.get("code")
352     if not code:
353         # we get here if we couldn't connect and there was nothing to unmarshal
354         die_bad_access("could not connect")
355
356     elif code == "stat":
357         expiry = result.get("TicketExpiration")
358         if expiry:
359             expiry = int(expiry)
360             if expiry > min_expiration:
361                 # ok to carry on
362                 return
363             else:
364                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
365
366         else:
367             # account without a timeout - all ok
368             return
369
370     elif code == "error":
371         data = result.get("data")
372         if data:
373             die_bad_access("p4 error: {0}".format(data))
374         else:
375             die_bad_access("unknown error")
376     elif code == "info":
377         return
378     else:
379         die_bad_access("unknown error code {0}".format(code))
380
381 _p4_version_string = None
382 def p4_version_string():
383     """Read the version string, showing just the last line, which
384        hopefully is the interesting version bit.
385
386        $ p4 -V
387        Perforce - The Fast Software Configuration Management System.
388        Copyright 1995-2011 Perforce Software.  All rights reserved.
389        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
390     """
391     global _p4_version_string
392     if not _p4_version_string:
393         a = p4_read_pipe_lines(["-V"])
394         _p4_version_string = a[-1].rstrip()
395     return _p4_version_string
396
397 def p4_integrate(src, dest):
398     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
399
400 def p4_sync(f, *options):
401     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
402
403 def p4_add(f):
404     # forcibly add file names with wildcards
405     if wildcard_present(f):
406         p4_system(["add", "-f", f])
407     else:
408         p4_system(["add", f])
409
410 def p4_delete(f):
411     p4_system(["delete", wildcard_encode(f)])
412
413 def p4_edit(f, *options):
414     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
415
416 def p4_revert(f):
417     p4_system(["revert", wildcard_encode(f)])
418
419 def p4_reopen(type, f):
420     p4_system(["reopen", "-t", type, wildcard_encode(f)])
421
422 def p4_reopen_in_change(changelist, files):
423     cmd = ["reopen", "-c", str(changelist)] + files
424     p4_system(cmd)
425
426 def p4_move(src, dest):
427     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
428
429 def p4_last_change():
430     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
431     return int(results[0]['change'])
432
433 def p4_describe(change, shelved=False):
434     """Make sure it returns a valid result by checking for
435        the presence of field "time".  Return a dict of the
436        results."""
437
438     cmd = ["describe", "-s"]
439     if shelved:
440         cmd += ["-S"]
441     cmd += [str(change)]
442
443     ds = p4CmdList(cmd, skip_info=True)
444     if len(ds) != 1:
445         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
446
447     d = ds[0]
448
449     if "p4ExitCode" in d:
450         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
451                                                       str(d)))
452     if "code" in d:
453         if d["code"] == "error":
454             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
455
456     if "time" not in d:
457         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
458
459     return d
460
461 #
462 # Canonicalize the p4 type and return a tuple of the
463 # base type, plus any modifiers.  See "p4 help filetypes"
464 # for a list and explanation.
465 #
466 def split_p4_type(p4type):
467
468     p4_filetypes_historical = {
469         "ctempobj": "binary+Sw",
470         "ctext": "text+C",
471         "cxtext": "text+Cx",
472         "ktext": "text+k",
473         "kxtext": "text+kx",
474         "ltext": "text+F",
475         "tempobj": "binary+FSw",
476         "ubinary": "binary+F",
477         "uresource": "resource+F",
478         "uxbinary": "binary+Fx",
479         "xbinary": "binary+x",
480         "xltext": "text+Fx",
481         "xtempobj": "binary+Swx",
482         "xtext": "text+x",
483         "xunicode": "unicode+x",
484         "xutf16": "utf16+x",
485     }
486     if p4type in p4_filetypes_historical:
487         p4type = p4_filetypes_historical[p4type]
488     mods = ""
489     s = p4type.split("+")
490     base = s[0]
491     mods = ""
492     if len(s) > 1:
493         mods = s[1]
494     return (base, mods)
495
496 #
497 # return the raw p4 type of a file (text, text+ko, etc)
498 #
499 def p4_type(f):
500     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
501     return results[0]['headType']
502
503 #
504 # Given a type base and modifier, return a regexp matching
505 # the keywords that can be expanded in the file
506 #
507 def p4_keywords_regexp_for_type(base, type_mods):
508     if base in ("text", "unicode", "binary"):
509         kwords = None
510         if "ko" in type_mods:
511             kwords = 'Id|Header'
512         elif "k" in type_mods:
513             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
514         else:
515             return None
516         pattern = r"""
517             \$              # Starts with a dollar, followed by...
518             (%s)            # one of the keywords, followed by...
519             (:[^$\n]+)?     # possibly an old expansion, followed by...
520             \$              # another dollar
521             """ % kwords
522         return pattern
523     else:
524         return None
525
526 #
527 # Given a file, return a regexp matching the possible
528 # RCS keywords that will be expanded, or None for files
529 # with kw expansion turned off.
530 #
531 def p4_keywords_regexp_for_file(file):
532     if not os.path.exists(file):
533         return None
534     else:
535         (type_base, type_mods) = split_p4_type(p4_type(file))
536         return p4_keywords_regexp_for_type(type_base, type_mods)
537
538 def setP4ExecBit(file, mode):
539     # Reopens an already open file and changes the execute bit to match
540     # the execute bit setting in the passed in mode.
541
542     p4Type = "+x"
543
544     if not isModeExec(mode):
545         p4Type = getP4OpenedType(file)
546         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
547         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
548         if p4Type[-1] == "+":
549             p4Type = p4Type[0:-1]
550
551     p4_reopen(p4Type, file)
552
553 def getP4OpenedType(file):
554     # Returns the perforce file type for the given file.
555
556     result = p4_read_pipe(["opened", wildcard_encode(file)])
557     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
558     if match:
559         return match.group(1)
560     else:
561         die("Could not determine file type for %s (result: '%s')" % (file, result))
562
563 # Return the set of all p4 labels
564 def getP4Labels(depotPaths):
565     labels = set()
566     if not isinstance(depotPaths, list):
567         depotPaths = [depotPaths]
568
569     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
570         label = l['label']
571         labels.add(label)
572
573     return labels
574
575 # Return the set of all git tags
576 def getGitTags():
577     gitTags = set()
578     for line in read_pipe_lines(["git", "tag"]):
579         tag = line.strip()
580         gitTags.add(tag)
581     return gitTags
582
583 _diff_tree_pattern = None
584
585 def parseDiffTreeEntry(entry):
586     """Parses a single diff tree entry into its component elements.
587
588     See git-diff-tree(1) manpage for details about the format of the diff
589     output. This method returns a dictionary with the following elements:
590
591     src_mode - The mode of the source file
592     dst_mode - The mode of the destination file
593     src_sha1 - The sha1 for the source file
594     dst_sha1 - The sha1 fr the destination file
595     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
596     status_score - The score for the status (applicable for 'C' and 'R'
597                    statuses). This is None if there is no score.
598     src - The path for the source file.
599     dst - The path for the destination file. This is only present for
600           copy or renames. If it is not present, this is None.
601
602     If the pattern is not matched, None is returned."""
603
604     global _diff_tree_pattern
605     if not _diff_tree_pattern:
606         _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
607
608     match = _diff_tree_pattern.match(entry)
609     if match:
610         return {
611             'src_mode': match.group(1),
612             'dst_mode': match.group(2),
613             'src_sha1': match.group(3),
614             'dst_sha1': match.group(4),
615             'status': match.group(5),
616             'status_score': match.group(6),
617             'src': match.group(7),
618             'dst': match.group(10)
619         }
620     return None
621
622 def isModeExec(mode):
623     # Returns True if the given git mode represents an executable file,
624     # otherwise False.
625     return mode[-3:] == "755"
626
627 class P4Exception(Exception):
628     """ Base class for exceptions from the p4 client """
629     def __init__(self, exit_code):
630         self.p4ExitCode = exit_code
631
632 class P4ServerException(P4Exception):
633     """ Base class for exceptions where we get some kind of marshalled up result from the server """
634     def __init__(self, exit_code, p4_result):
635         super(P4ServerException, self).__init__(exit_code)
636         self.p4_result = p4_result
637         self.code = p4_result[0]['code']
638         self.data = p4_result[0]['data']
639
640 class P4RequestSizeException(P4ServerException):
641     """ One of the maxresults or maxscanrows errors """
642     def __init__(self, exit_code, p4_result, limit):
643         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
644         self.limit = limit
645
646 class P4CommandException(P4Exception):
647     """ Something went wrong calling p4 which means we have to give up """
648     def __init__(self, msg):
649         self.msg = msg
650
651     def __str__(self):
652         return self.msg
653
654 def isModeExecChanged(src_mode, dst_mode):
655     return isModeExec(src_mode) != isModeExec(dst_mode)
656
657 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
658         errors_as_exceptions=False):
659
660     if not isinstance(cmd, list):
661         cmd = "-G " + cmd
662         expand = True
663     else:
664         cmd = ["-G"] + cmd
665         expand = False
666
667     cmd = p4_build_cmd(cmd)
668     if verbose:
669         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
670
671     # Use a temporary file to avoid deadlocks without
672     # subprocess.communicate(), which would put another copy
673     # of stdout into memory.
674     stdin_file = None
675     if stdin is not None:
676         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
677         if not isinstance(stdin, list):
678             stdin_file.write(stdin)
679         else:
680             for i in stdin:
681                 stdin_file.write(encode_text_stream(i))
682                 stdin_file.write(b'\n')
683         stdin_file.flush()
684         stdin_file.seek(0)
685
686     p4 = subprocess.Popen(cmd,
687                           shell=expand,
688                           stdin=stdin_file,
689                           stdout=subprocess.PIPE)
690
691     result = []
692     try:
693         while True:
694             entry = marshal.load(p4.stdout)
695             if bytes is not str:
696                 # Decode unmarshalled dict to use str keys and values, except for:
697                 #   - `data` which may contain arbitrary binary data
698                 #   - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
699                 decoded_entry = {}
700                 for key, value in entry.items():
701                     key = key.decode()
702                     if isinstance(value, bytes) and not (key in ('data', 'path', 'clientFile') or key.startswith('depotFile')):
703                         value = value.decode()
704                     decoded_entry[key] = value
705                 # Parse out data if it's an error response
706                 if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
707                     decoded_entry['data'] = decoded_entry['data'].decode()
708                 entry = decoded_entry
709             if skip_info:
710                 if 'code' in entry and entry['code'] == 'info':
711                     continue
712             if cb is not None:
713                 cb(entry)
714             else:
715                 result.append(entry)
716     except EOFError:
717         pass
718     exitCode = p4.wait()
719     if exitCode != 0:
720         if errors_as_exceptions:
721             if len(result) > 0:
722                 data = result[0].get('data')
723                 if data:
724                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
725                     if not m:
726                         m = re.search('Request too large \(over (\d+)\)', data)
727
728                     if m:
729                         limit = int(m.group(1))
730                         raise P4RequestSizeException(exitCode, result, limit)
731
732                 raise P4ServerException(exitCode, result)
733             else:
734                 raise P4Exception(exitCode)
735         else:
736             entry = {}
737             entry["p4ExitCode"] = exitCode
738             result.append(entry)
739
740     return result
741
742 def p4Cmd(cmd):
743     list = p4CmdList(cmd)
744     result = {}
745     for entry in list:
746         result.update(entry)
747     return result;
748
749 def p4Where(depotPath):
750     if not depotPath.endswith("/"):
751         depotPath += "/"
752     depotPathLong = depotPath + "..."
753     outputList = p4CmdList(["where", depotPathLong])
754     output = None
755     for entry in outputList:
756         if "depotFile" in entry:
757             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
758             # The base path always ends with "/...".
759             entry_path = decode_path(entry['depotFile'])
760             if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
761                 output = entry
762                 break
763         elif "data" in entry:
764             data = entry.get("data")
765             space = data.find(" ")
766             if data[:space] == depotPath:
767                 output = entry
768                 break
769     if output == None:
770         return ""
771     if output["code"] == "error":
772         return ""
773     clientPath = ""
774     if "path" in output:
775         clientPath = decode_path(output['path'])
776     elif "data" in output:
777         data = output.get("data")
778         lastSpace = data.rfind(b" ")
779         clientPath = decode_path(data[lastSpace + 1:])
780
781     if clientPath.endswith("..."):
782         clientPath = clientPath[:-3]
783     return clientPath
784
785 def currentGitBranch():
786     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
787
788 def isValidGitDir(path):
789     return git_dir(path) != None
790
791 def parseRevision(ref):
792     return read_pipe("git rev-parse %s" % ref).strip()
793
794 def branchExists(ref):
795     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
796                      ignore_error=True)
797     return len(rev) > 0
798
799 def extractLogMessageFromGitCommit(commit):
800     logMessage = ""
801
802     ## fixme: title is first line of commit, not 1st paragraph.
803     foundTitle = False
804     for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
805        if not foundTitle:
806            if len(log) == 1:
807                foundTitle = True
808            continue
809
810        logMessage += log
811     return logMessage
812
813 def extractSettingsGitLog(log):
814     values = {}
815     for line in log.split("\n"):
816         line = line.strip()
817         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
818         if not m:
819             continue
820
821         assignments = m.group(1).split (':')
822         for a in assignments:
823             vals = a.split ('=')
824             key = vals[0].strip()
825             val = ('='.join (vals[1:])).strip()
826             if val.endswith ('\"') and val.startswith('"'):
827                 val = val[1:-1]
828
829             values[key] = val
830
831     paths = values.get("depot-paths")
832     if not paths:
833         paths = values.get("depot-path")
834     if paths:
835         values['depot-paths'] = paths.split(',')
836     return values
837
838 def gitBranchExists(branch):
839     proc = subprocess.Popen(["git", "rev-parse", branch],
840                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
841     return proc.wait() == 0;
842
843 def gitUpdateRef(ref, newvalue):
844     subprocess.check_call(["git", "update-ref", ref, newvalue])
845
846 def gitDeleteRef(ref):
847     subprocess.check_call(["git", "update-ref", "-d", ref])
848
849 _gitConfig = {}
850
851 def gitConfig(key, typeSpecifier=None):
852     if key not in _gitConfig:
853         cmd = [ "git", "config" ]
854         if typeSpecifier:
855             cmd += [ typeSpecifier ]
856         cmd += [ key ]
857         s = read_pipe(cmd, ignore_error=True)
858         _gitConfig[key] = s.strip()
859     return _gitConfig[key]
860
861 def gitConfigBool(key):
862     """Return a bool, using git config --bool.  It is True only if the
863        variable is set to true, and False if set to false or not present
864        in the config."""
865
866     if key not in _gitConfig:
867         _gitConfig[key] = gitConfig(key, '--bool') == "true"
868     return _gitConfig[key]
869
870 def gitConfigInt(key):
871     if key not in _gitConfig:
872         cmd = [ "git", "config", "--int", key ]
873         s = read_pipe(cmd, ignore_error=True)
874         v = s.strip()
875         try:
876             _gitConfig[key] = int(gitConfig(key, '--int'))
877         except ValueError:
878             _gitConfig[key] = None
879     return _gitConfig[key]
880
881 def gitConfigList(key):
882     if key not in _gitConfig:
883         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
884         _gitConfig[key] = s.strip().splitlines()
885         if _gitConfig[key] == ['']:
886             _gitConfig[key] = []
887     return _gitConfig[key]
888
889 def p4BranchesInGit(branchesAreInRemotes=True):
890     """Find all the branches whose names start with "p4/", looking
891        in remotes or heads as specified by the argument.  Return
892        a dictionary of { branch: revision } for each one found.
893        The branch names are the short names, without any
894        "p4/" prefix."""
895
896     branches = {}
897
898     cmdline = "git rev-parse --symbolic "
899     if branchesAreInRemotes:
900         cmdline += "--remotes"
901     else:
902         cmdline += "--branches"
903
904     for line in read_pipe_lines(cmdline):
905         line = line.strip()
906
907         # only import to p4/
908         if not line.startswith('p4/'):
909             continue
910         # special symbolic ref to p4/master
911         if line == "p4/HEAD":
912             continue
913
914         # strip off p4/ prefix
915         branch = line[len("p4/"):]
916
917         branches[branch] = parseRevision(line)
918
919     return branches
920
921 def branch_exists(branch):
922     """Make sure that the given ref name really exists."""
923
924     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
925     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
926     out, _ = p.communicate()
927     out = decode_text_stream(out)
928     if p.returncode:
929         return False
930     # expect exactly one line of output: the branch name
931     return out.rstrip() == branch
932
933 def findUpstreamBranchPoint(head = "HEAD"):
934     branches = p4BranchesInGit()
935     # map from depot-path to branch name
936     branchByDepotPath = {}
937     for branch in branches.keys():
938         tip = branches[branch]
939         log = extractLogMessageFromGitCommit(tip)
940         settings = extractSettingsGitLog(log)
941         if "depot-paths" in settings:
942             paths = ",".join(settings["depot-paths"])
943             branchByDepotPath[paths] = "remotes/p4/" + branch
944
945     settings = None
946     parent = 0
947     while parent < 65535:
948         commit = head + "~%s" % parent
949         log = extractLogMessageFromGitCommit(commit)
950         settings = extractSettingsGitLog(log)
951         if "depot-paths" in settings:
952             paths = ",".join(settings["depot-paths"])
953             if paths in branchByDepotPath:
954                 return [branchByDepotPath[paths], settings]
955
956         parent = parent + 1
957
958     return ["", settings]
959
960 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
961     if not silent:
962         print("Creating/updating branch(es) in %s based on origin branch(es)"
963                % localRefPrefix)
964
965     originPrefix = "origin/p4/"
966
967     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
968         line = line.strip()
969         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
970             continue
971
972         headName = line[len(originPrefix):]
973         remoteHead = localRefPrefix + headName
974         originHead = line
975
976         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
977         if ('depot-paths' not in original
978             or 'change' not in original):
979             continue
980
981         update = False
982         if not gitBranchExists(remoteHead):
983             if verbose:
984                 print("creating %s" % remoteHead)
985             update = True
986         else:
987             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
988             if 'change' in settings:
989                 if settings['depot-paths'] == original['depot-paths']:
990                     originP4Change = int(original['change'])
991                     p4Change = int(settings['change'])
992                     if originP4Change > p4Change:
993                         print("%s (%s) is newer than %s (%s). "
994                                "Updating p4 branch from origin."
995                                % (originHead, originP4Change,
996                                   remoteHead, p4Change))
997                         update = True
998                 else:
999                     print("Ignoring: %s was imported from %s while "
1000                            "%s was imported from %s"
1001                            % (originHead, ','.join(original['depot-paths']),
1002                               remoteHead, ','.join(settings['depot-paths'])))
1003
1004         if update:
1005             system("git update-ref %s %s" % (remoteHead, originHead))
1006
1007 def originP4BranchesExist():
1008         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1009
1010
1011 def p4ParseNumericChangeRange(parts):
1012     changeStart = int(parts[0][1:])
1013     if parts[1] == '#head':
1014         changeEnd = p4_last_change()
1015     else:
1016         changeEnd = int(parts[1])
1017
1018     return (changeStart, changeEnd)
1019
1020 def chooseBlockSize(blockSize):
1021     if blockSize:
1022         return blockSize
1023     else:
1024         return defaultBlockSize
1025
1026 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
1027     assert depotPaths
1028
1029     # Parse the change range into start and end. Try to find integer
1030     # revision ranges as these can be broken up into blocks to avoid
1031     # hitting server-side limits (maxrows, maxscanresults). But if
1032     # that doesn't work, fall back to using the raw revision specifier
1033     # strings, without using block mode.
1034
1035     if changeRange is None or changeRange == '':
1036         changeStart = 1
1037         changeEnd = p4_last_change()
1038         block_size = chooseBlockSize(requestedBlockSize)
1039     else:
1040         parts = changeRange.split(',')
1041         assert len(parts) == 2
1042         try:
1043             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
1044             block_size = chooseBlockSize(requestedBlockSize)
1045         except ValueError:
1046             changeStart = parts[0][1:]
1047             changeEnd = parts[1]
1048             if requestedBlockSize:
1049                 die("cannot use --changes-block-size with non-numeric revisions")
1050             block_size = None
1051
1052     changes = set()
1053
1054     # Retrieve changes a block at a time, to prevent running
1055     # into a MaxResults/MaxScanRows error from the server. If
1056     # we _do_ hit one of those errors, turn down the block size
1057
1058     while True:
1059         cmd = ['changes']
1060
1061         if block_size:
1062             end = min(changeEnd, changeStart + block_size)
1063             revisionRange = "%d,%d" % (changeStart, end)
1064         else:
1065             revisionRange = "%s,%s" % (changeStart, changeEnd)
1066
1067         for p in depotPaths:
1068             cmd += ["%s...@%s" % (p, revisionRange)]
1069
1070         # fetch the changes
1071         try:
1072             result = p4CmdList(cmd, errors_as_exceptions=True)
1073         except P4RequestSizeException as e:
1074             if not block_size:
1075                 block_size = e.limit
1076             elif block_size > e.limit:
1077                 block_size = e.limit
1078             else:
1079                 block_size = max(2, block_size // 2)
1080
1081             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1082             continue
1083         except P4Exception as e:
1084             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1085
1086         # Insert changes in chronological order
1087         for entry in reversed(result):
1088             if 'change' not in entry:
1089                 continue
1090             changes.add(int(entry['change']))
1091
1092         if not block_size:
1093             break
1094
1095         if end >= changeEnd:
1096             break
1097
1098         changeStart = end + 1
1099
1100     changes = sorted(changes)
1101     return changes
1102
1103 def p4PathStartsWith(path, prefix):
1104     # This method tries to remedy a potential mixed-case issue:
1105     #
1106     # If UserA adds  //depot/DirA/file1
1107     # and UserB adds //depot/dira/file2
1108     #
1109     # we may or may not have a problem. If you have core.ignorecase=true,
1110     # we treat DirA and dira as the same directory
1111     if gitConfigBool("core.ignorecase"):
1112         return path.lower().startswith(prefix.lower())
1113     return path.startswith(prefix)
1114
1115 def getClientSpec():
1116     """Look at the p4 client spec, create a View() object that contains
1117        all the mappings, and return it."""
1118
1119     specList = p4CmdList("client -o")
1120     if len(specList) != 1:
1121         die('Output from "client -o" is %d lines, expecting 1' %
1122             len(specList))
1123
1124     # dictionary of all client parameters
1125     entry = specList[0]
1126
1127     # the //client/ name
1128     client_name = entry["Client"]
1129
1130     # just the keys that start with "View"
1131     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1132
1133     # hold this new View
1134     view = View(client_name)
1135
1136     # append the lines, in order, to the view
1137     for view_num in range(len(view_keys)):
1138         k = "View%d" % view_num
1139         if k not in view_keys:
1140             die("Expected view key %s missing" % k)
1141         view.append(entry[k])
1142
1143     return view
1144
1145 def getClientRoot():
1146     """Grab the client directory."""
1147
1148     output = p4CmdList("client -o")
1149     if len(output) != 1:
1150         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1151
1152     entry = output[0]
1153     if "Root" not in entry:
1154         die('Client has no "Root"')
1155
1156     return entry["Root"]
1157
1158 #
1159 # P4 wildcards are not allowed in filenames.  P4 complains
1160 # if you simply add them, but you can force it with "-f", in
1161 # which case it translates them into %xx encoding internally.
1162 #
1163 def wildcard_decode(path):
1164     # Search for and fix just these four characters.  Do % last so
1165     # that fixing it does not inadvertently create new %-escapes.
1166     # Cannot have * in a filename in windows; untested as to
1167     # what p4 would do in such a case.
1168     if not platform.system() == "Windows":
1169         path = path.replace("%2A", "*")
1170     path = path.replace("%23", "#") \
1171                .replace("%40", "@") \
1172                .replace("%25", "%")
1173     return path
1174
1175 def wildcard_encode(path):
1176     # do % first to avoid double-encoding the %s introduced here
1177     path = path.replace("%", "%25") \
1178                .replace("*", "%2A") \
1179                .replace("#", "%23") \
1180                .replace("@", "%40")
1181     return path
1182
1183 def wildcard_present(path):
1184     m = re.search("[*#@%]", path)
1185     return m is not None
1186
1187 class LargeFileSystem(object):
1188     """Base class for large file system support."""
1189
1190     def __init__(self, writeToGitStream):
1191         self.largeFiles = set()
1192         self.writeToGitStream = writeToGitStream
1193
1194     def generatePointer(self, cloneDestination, contentFile):
1195         """Return the content of a pointer file that is stored in Git instead of
1196            the actual content."""
1197         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1198
1199     def pushFile(self, localLargeFile):
1200         """Push the actual content which is not stored in the Git repository to
1201            a server."""
1202         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1203
1204     def hasLargeFileExtension(self, relPath):
1205         return functools.reduce(
1206             lambda a, b: a or b,
1207             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1208             False
1209         )
1210
1211     def generateTempFile(self, contents):
1212         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1213         for d in contents:
1214             contentFile.write(d)
1215         contentFile.close()
1216         return contentFile.name
1217
1218     def exceedsLargeFileThreshold(self, relPath, contents):
1219         if gitConfigInt('git-p4.largeFileThreshold'):
1220             contentsSize = sum(len(d) for d in contents)
1221             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1222                 return True
1223         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1224             contentsSize = sum(len(d) for d in contents)
1225             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1226                 return False
1227             contentTempFile = self.generateTempFile(contents)
1228             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1229             with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1230                 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1231                 compressedContentsSize = zf.infolist()[0].compress_size
1232             os.remove(contentTempFile)
1233             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1234                 return True
1235         return False
1236
1237     def addLargeFile(self, relPath):
1238         self.largeFiles.add(relPath)
1239
1240     def removeLargeFile(self, relPath):
1241         self.largeFiles.remove(relPath)
1242
1243     def isLargeFile(self, relPath):
1244         return relPath in self.largeFiles
1245
1246     def processContent(self, git_mode, relPath, contents):
1247         """Processes the content of git fast import. This method decides if a
1248            file is stored in the large file system and handles all necessary
1249            steps."""
1250         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1251             contentTempFile = self.generateTempFile(contents)
1252             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1253             if pointer_git_mode:
1254                 git_mode = pointer_git_mode
1255             if localLargeFile:
1256                 # Move temp file to final location in large file system
1257                 largeFileDir = os.path.dirname(localLargeFile)
1258                 if not os.path.isdir(largeFileDir):
1259                     os.makedirs(largeFileDir)
1260                 shutil.move(contentTempFile, localLargeFile)
1261                 self.addLargeFile(relPath)
1262                 if gitConfigBool('git-p4.largeFilePush'):
1263                     self.pushFile(localLargeFile)
1264                 if verbose:
1265                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1266         return (git_mode, contents)
1267
1268 class MockLFS(LargeFileSystem):
1269     """Mock large file system for testing."""
1270
1271     def generatePointer(self, contentFile):
1272         """The pointer content is the original content prefixed with "pointer-".
1273            The local filename of the large file storage is derived from the file content.
1274            """
1275         with open(contentFile, 'r') as f:
1276             content = next(f)
1277             gitMode = '100644'
1278             pointerContents = 'pointer-' + content
1279             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1280             return (gitMode, pointerContents, localLargeFile)
1281
1282     def pushFile(self, localLargeFile):
1283         """The remote filename of the large file storage is the same as the local
1284            one but in a different directory.
1285            """
1286         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1287         if not os.path.exists(remotePath):
1288             os.makedirs(remotePath)
1289         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1290
1291 class GitLFS(LargeFileSystem):
1292     """Git LFS as backend for the git-p4 large file system.
1293        See https://git-lfs.github.com/ for details."""
1294
1295     def __init__(self, *args):
1296         LargeFileSystem.__init__(self, *args)
1297         self.baseGitAttributes = []
1298
1299     def generatePointer(self, contentFile):
1300         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1301            mode and content which is stored in the Git repository instead of
1302            the actual content. Return also the new location of the actual
1303            content.
1304            """
1305         if os.path.getsize(contentFile) == 0:
1306             return (None, '', None)
1307
1308         pointerProcess = subprocess.Popen(
1309             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1310             stdout=subprocess.PIPE
1311         )
1312         pointerFile = decode_text_stream(pointerProcess.stdout.read())
1313         if pointerProcess.wait():
1314             os.remove(contentFile)
1315             die('git-lfs pointer command failed. Did you install the extension?')
1316
1317         # Git LFS removed the preamble in the output of the 'pointer' command
1318         # starting from version 1.2.0. Check for the preamble here to support
1319         # earlier versions.
1320         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1321         if pointerFile.startswith('Git LFS pointer for'):
1322             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1323
1324         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1325         # if someone use external lfs.storage ( not in local repo git )
1326         lfs_path = gitConfig('lfs.storage')
1327         if not lfs_path:
1328             lfs_path = 'lfs'
1329         if not os.path.isabs(lfs_path):
1330             lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1331         localLargeFile = os.path.join(
1332             lfs_path,
1333             'objects', oid[:2], oid[2:4],
1334             oid,
1335         )
1336         # LFS Spec states that pointer files should not have the executable bit set.
1337         gitMode = '100644'
1338         return (gitMode, pointerFile, localLargeFile)
1339
1340     def pushFile(self, localLargeFile):
1341         uploadProcess = subprocess.Popen(
1342             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1343         )
1344         if uploadProcess.wait():
1345             die('git-lfs push command failed. Did you define a remote?')
1346
1347     def generateGitAttributes(self):
1348         return (
1349             self.baseGitAttributes +
1350             [
1351                 '\n',
1352                 '#\n',
1353                 '# Git LFS (see https://git-lfs.github.com/)\n',
1354                 '#\n',
1355             ] +
1356             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1357                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1358             ] +
1359             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1360                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1361             ]
1362         )
1363
1364     def addLargeFile(self, relPath):
1365         LargeFileSystem.addLargeFile(self, relPath)
1366         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1367
1368     def removeLargeFile(self, relPath):
1369         LargeFileSystem.removeLargeFile(self, relPath)
1370         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1371
1372     def processContent(self, git_mode, relPath, contents):
1373         if relPath == '.gitattributes':
1374             self.baseGitAttributes = contents
1375             return (git_mode, self.generateGitAttributes())
1376         else:
1377             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1378
1379 class Command:
1380     delete_actions = ( "delete", "move/delete", "purge" )
1381     add_actions = ( "add", "branch", "move/add" )
1382
1383     def __init__(self):
1384         self.usage = "usage: %prog [options]"
1385         self.needsGit = True
1386         self.verbose = False
1387
1388     # This is required for the "append" update_shelve action
1389     def ensure_value(self, attr, value):
1390         if not hasattr(self, attr) or getattr(self, attr) is None:
1391             setattr(self, attr, value)
1392         return getattr(self, attr)
1393
1394 class P4UserMap:
1395     def __init__(self):
1396         self.userMapFromPerforceServer = False
1397         self.myP4UserId = None
1398
1399     def p4UserId(self):
1400         if self.myP4UserId:
1401             return self.myP4UserId
1402
1403         results = p4CmdList("user -o")
1404         for r in results:
1405             if 'User' in r:
1406                 self.myP4UserId = r['User']
1407                 return r['User']
1408         die("Could not find your p4 user id")
1409
1410     def p4UserIsMe(self, p4User):
1411         # return True if the given p4 user is actually me
1412         me = self.p4UserId()
1413         if not p4User or p4User != me:
1414             return False
1415         else:
1416             return True
1417
1418     def getUserCacheFilename(self):
1419         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1420         return home + "/.gitp4-usercache.txt"
1421
1422     def getUserMapFromPerforceServer(self):
1423         if self.userMapFromPerforceServer:
1424             return
1425         self.users = {}
1426         self.emails = {}
1427
1428         for output in p4CmdList("users"):
1429             if "User" not in output:
1430                 continue
1431             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1432             self.emails[output["Email"]] = output["User"]
1433
1434         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1435         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1436             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1437             if mapUser and len(mapUser[0]) == 3:
1438                 user = mapUser[0][0]
1439                 fullname = mapUser[0][1]
1440                 email = mapUser[0][2]
1441                 self.users[user] = fullname + " <" + email + ">"
1442                 self.emails[email] = user
1443
1444         s = ''
1445         for (key, val) in self.users.items():
1446             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1447
1448         open(self.getUserCacheFilename(), 'w').write(s)
1449         self.userMapFromPerforceServer = True
1450
1451     def loadUserMapFromCache(self):
1452         self.users = {}
1453         self.userMapFromPerforceServer = False
1454         try:
1455             cache = open(self.getUserCacheFilename(), 'r')
1456             lines = cache.readlines()
1457             cache.close()
1458             for line in lines:
1459                 entry = line.strip().split("\t")
1460                 self.users[entry[0]] = entry[1]
1461         except IOError:
1462             self.getUserMapFromPerforceServer()
1463
1464 class P4Debug(Command):
1465     def __init__(self):
1466         Command.__init__(self)
1467         self.options = []
1468         self.description = "A tool to debug the output of p4 -G."
1469         self.needsGit = False
1470
1471     def run(self, args):
1472         j = 0
1473         for output in p4CmdList(args):
1474             print('Element: %d' % j)
1475             j += 1
1476             print(output)
1477         return True
1478
1479 class P4RollBack(Command):
1480     def __init__(self):
1481         Command.__init__(self)
1482         self.options = [
1483             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1484         ]
1485         self.description = "A tool to debug the multi-branch import. Don't use :)"
1486         self.rollbackLocalBranches = False
1487
1488     def run(self, args):
1489         if len(args) != 1:
1490             return False
1491         maxChange = int(args[0])
1492
1493         if "p4ExitCode" in p4Cmd("changes -m 1"):
1494             die("Problems executing p4");
1495
1496         if self.rollbackLocalBranches:
1497             refPrefix = "refs/heads/"
1498             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1499         else:
1500             refPrefix = "refs/remotes/"
1501             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1502
1503         for line in lines:
1504             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1505                 line = line.strip()
1506                 ref = refPrefix + line
1507                 log = extractLogMessageFromGitCommit(ref)
1508                 settings = extractSettingsGitLog(log)
1509
1510                 depotPaths = settings['depot-paths']
1511                 change = settings['change']
1512
1513                 changed = False
1514
1515                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1516                                                            for p in depotPaths]))) == 0:
1517                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1518                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1519                     continue
1520
1521                 while change and int(change) > maxChange:
1522                     changed = True
1523                     if self.verbose:
1524                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1525                     system("git update-ref %s \"%s^\"" % (ref, ref))
1526                     log = extractLogMessageFromGitCommit(ref)
1527                     settings =  extractSettingsGitLog(log)
1528
1529
1530                     depotPaths = settings['depot-paths']
1531                     change = settings['change']
1532
1533                 if changed:
1534                     print("%s rewound to %s" % (ref, change))
1535
1536         return True
1537
1538 class P4Submit(Command, P4UserMap):
1539
1540     conflict_behavior_choices = ("ask", "skip", "quit")
1541
1542     def __init__(self):
1543         Command.__init__(self)
1544         P4UserMap.__init__(self)
1545         self.options = [
1546                 optparse.make_option("--origin", dest="origin"),
1547                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1548                 # preserve the user, requires relevant p4 permissions
1549                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1550                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1551                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1552                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1553                 optparse.make_option("--conflict", dest="conflict_behavior",
1554                                      choices=self.conflict_behavior_choices),
1555                 optparse.make_option("--branch", dest="branch"),
1556                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1557                                      help="Shelve instead of submit. Shelved files are reverted, "
1558                                      "restoring the workspace to the state before the shelve"),
1559                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1560                                      metavar="CHANGELIST",
1561                                      help="update an existing shelved changelist, implies --shelve, "
1562                                            "repeat in-order for multiple shelved changelists"),
1563                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1564                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1565                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1566                                      help="Disable rebase after submit is completed. Can be useful if you "
1567                                      "work from a local git branch that is not master"),
1568                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1569                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1570         ]
1571         self.description = """Submit changes from git to the perforce depot.\n
1572     The `p4-pre-submit` hook is executed if it exists and is executable.
1573     The hook takes no parameters and nothing from standard input. Exiting with
1574     non-zero status from this script prevents `git-p4 submit` from launching.
1575
1576     One usage scenario is to run unit tests in the hook."""
1577
1578         self.usage += " [name of git branch to submit into perforce depot]"
1579         self.origin = ""
1580         self.detectRenames = False
1581         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1582         self.dry_run = False
1583         self.shelve = False
1584         self.update_shelve = list()
1585         self.commit = ""
1586         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1587         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1588         self.prepare_p4_only = False
1589         self.conflict_behavior = None
1590         self.isWindows = (platform.system() == "Windows")
1591         self.exportLabels = False
1592         self.p4HasMoveCommand = p4_has_move_command()
1593         self.branch = None
1594
1595         if gitConfig('git-p4.largeFileSystem'):
1596             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1597
1598     def check(self):
1599         if len(p4CmdList("opened ...")) > 0:
1600             die("You have files opened with perforce! Close them before starting the sync.")
1601
1602     def separate_jobs_from_description(self, message):
1603         """Extract and return a possible Jobs field in the commit
1604            message.  It goes into a separate section in the p4 change
1605            specification.
1606
1607            A jobs line starts with "Jobs:" and looks like a new field
1608            in a form.  Values are white-space separated on the same
1609            line or on following lines that start with a tab.
1610
1611            This does not parse and extract the full git commit message
1612            like a p4 form.  It just sees the Jobs: line as a marker
1613            to pass everything from then on directly into the p4 form,
1614            but outside the description section.
1615
1616            Return a tuple (stripped log message, jobs string)."""
1617
1618         m = re.search(r'^Jobs:', message, re.MULTILINE)
1619         if m is None:
1620             return (message, None)
1621
1622         jobtext = message[m.start():]
1623         stripped_message = message[:m.start()].rstrip()
1624         return (stripped_message, jobtext)
1625
1626     def prepareLogMessage(self, template, message, jobs):
1627         """Edits the template returned from "p4 change -o" to insert
1628            the message in the Description field, and the jobs text in
1629            the Jobs field."""
1630         result = ""
1631
1632         inDescriptionSection = False
1633
1634         for line in template.split("\n"):
1635             if line.startswith("#"):
1636                 result += line + "\n"
1637                 continue
1638
1639             if inDescriptionSection:
1640                 if line.startswith("Files:") or line.startswith("Jobs:"):
1641                     inDescriptionSection = False
1642                     # insert Jobs section
1643                     if jobs:
1644                         result += jobs + "\n"
1645                 else:
1646                     continue
1647             else:
1648                 if line.startswith("Description:"):
1649                     inDescriptionSection = True
1650                     line += "\n"
1651                     for messageLine in message.split("\n"):
1652                         line += "\t" + messageLine + "\n"
1653
1654             result += line + "\n"
1655
1656         return result
1657
1658     def patchRCSKeywords(self, file, pattern):
1659         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1660         (handle, outFileName) = tempfile.mkstemp(dir='.')
1661         try:
1662             outFile = os.fdopen(handle, "w+")
1663             inFile = open(file, "r")
1664             regexp = re.compile(pattern, re.VERBOSE)
1665             for line in inFile.readlines():
1666                 line = regexp.sub(r'$\1$', line)
1667                 outFile.write(line)
1668             inFile.close()
1669             outFile.close()
1670             # Forcibly overwrite the original file
1671             os.unlink(file)
1672             shutil.move(outFileName, file)
1673         except:
1674             # cleanup our temporary file
1675             os.unlink(outFileName)
1676             print("Failed to strip RCS keywords in %s" % file)
1677             raise
1678
1679         print("Patched up RCS keywords in %s" % file)
1680
1681     def p4UserForCommit(self,id):
1682         # Return the tuple (perforce user,git email) for a given git commit id
1683         self.getUserMapFromPerforceServer()
1684         gitEmail = read_pipe(["git", "log", "--max-count=1",
1685                               "--format=%ae", id])
1686         gitEmail = gitEmail.strip()
1687         if gitEmail not in self.emails:
1688             return (None,gitEmail)
1689         else:
1690             return (self.emails[gitEmail],gitEmail)
1691
1692     def checkValidP4Users(self,commits):
1693         # check if any git authors cannot be mapped to p4 users
1694         for id in commits:
1695             (user,email) = self.p4UserForCommit(id)
1696             if not user:
1697                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1698                 if gitConfigBool("git-p4.allowMissingP4Users"):
1699                     print("%s" % msg)
1700                 else:
1701                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1702
1703     def lastP4Changelist(self):
1704         # Get back the last changelist number submitted in this client spec. This
1705         # then gets used to patch up the username in the change. If the same
1706         # client spec is being used by multiple processes then this might go
1707         # wrong.
1708         results = p4CmdList("client -o")        # find the current client
1709         client = None
1710         for r in results:
1711             if 'Client' in r:
1712                 client = r['Client']
1713                 break
1714         if not client:
1715             die("could not get client spec")
1716         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1717         for r in results:
1718             if 'change' in r:
1719                 return r['change']
1720         die("Could not get changelist number for last submit - cannot patch up user details")
1721
1722     def modifyChangelistUser(self, changelist, newUser):
1723         # fixup the user field of a changelist after it has been submitted.
1724         changes = p4CmdList("change -o %s" % changelist)
1725         if len(changes) != 1:
1726             die("Bad output from p4 change modifying %s to user %s" %
1727                 (changelist, newUser))
1728
1729         c = changes[0]
1730         if c['User'] == newUser: return   # nothing to do
1731         c['User'] = newUser
1732         # p4 does not understand format version 3 and above
1733         input = marshal.dumps(c, 2)
1734
1735         result = p4CmdList("change -f -i", stdin=input)
1736         for r in result:
1737             if 'code' in r:
1738                 if r['code'] == 'error':
1739                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1740             if 'data' in r:
1741                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1742                 return
1743         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1744
1745     def canChangeChangelists(self):
1746         # check to see if we have p4 admin or super-user permissions, either of
1747         # which are required to modify changelists.
1748         results = p4CmdList(["protects", self.depotPath])
1749         for r in results:
1750             if 'perm' in r:
1751                 if r['perm'] == 'admin':
1752                     return 1
1753                 if r['perm'] == 'super':
1754                     return 1
1755         return 0
1756
1757     def prepareSubmitTemplate(self, changelist=None):
1758         """Run "p4 change -o" to grab a change specification template.
1759            This does not use "p4 -G", as it is nice to keep the submission
1760            template in original order, since a human might edit it.
1761
1762            Remove lines in the Files section that show changes to files
1763            outside the depot path we're committing into."""
1764
1765         [upstream, settings] = findUpstreamBranchPoint()
1766
1767         template = """\
1768 # A Perforce Change Specification.
1769 #
1770 #  Change:      The change number. 'new' on a new changelist.
1771 #  Date:        The date this specification was last modified.
1772 #  Client:      The client on which the changelist was created.  Read-only.
1773 #  User:        The user who created the changelist.
1774 #  Status:      Either 'pending' or 'submitted'. Read-only.
1775 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1776 #  Description: Comments about the changelist.  Required.
1777 #  Jobs:        What opened jobs are to be closed by this changelist.
1778 #               You may delete jobs from this list.  (New changelists only.)
1779 #  Files:       What opened files from the default changelist are to be added
1780 #               to this changelist.  You may delete files from this list.
1781 #               (New changelists only.)
1782 """
1783         files_list = []
1784         inFilesSection = False
1785         change_entry = None
1786         args = ['change', '-o']
1787         if changelist:
1788             args.append(str(changelist))
1789         for entry in p4CmdList(args):
1790             if 'code' not in entry:
1791                 continue
1792             if entry['code'] == 'stat':
1793                 change_entry = entry
1794                 break
1795         if not change_entry:
1796             die('Failed to decode output of p4 change -o')
1797         for key, value in change_entry.items():
1798             if key.startswith('File'):
1799                 if 'depot-paths' in settings:
1800                     if not [p for p in settings['depot-paths']
1801                             if p4PathStartsWith(value, p)]:
1802                         continue
1803                 else:
1804                     if not p4PathStartsWith(value, self.depotPath):
1805                         continue
1806                 files_list.append(value)
1807                 continue
1808         # Output in the order expected by prepareLogMessage
1809         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1810             if key not in change_entry:
1811                 continue
1812             template += '\n'
1813             template += key + ':'
1814             if key == 'Description':
1815                 template += '\n'
1816             for field_line in change_entry[key].splitlines():
1817                 template += '\t'+field_line+'\n'
1818         if len(files_list) > 0:
1819             template += '\n'
1820             template += 'Files:\n'
1821         for path in files_list:
1822             template += '\t'+path+'\n'
1823         return template
1824
1825     def edit_template(self, template_file):
1826         """Invoke the editor to let the user change the submission
1827            message.  Return true if okay to continue with the submit."""
1828
1829         # if configured to skip the editing part, just submit
1830         if gitConfigBool("git-p4.skipSubmitEdit"):
1831             return True
1832
1833         # look at the modification time, to check later if the user saved
1834         # the file
1835         mtime = os.stat(template_file).st_mtime
1836
1837         # invoke the editor
1838         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1839             editor = os.environ.get("P4EDITOR")
1840         else:
1841             editor = read_pipe("git var GIT_EDITOR").strip()
1842         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1843
1844         # If the file was not saved, prompt to see if this patch should
1845         # be skipped.  But skip this verification step if configured so.
1846         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1847             return True
1848
1849         # modification time updated means user saved the file
1850         if os.stat(template_file).st_mtime > mtime:
1851             return True
1852
1853         response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1854         if response == 'y':
1855             return True
1856         if response == 'n':
1857             return False
1858
1859     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1860         # diff
1861         if "P4DIFF" in os.environ:
1862             del(os.environ["P4DIFF"])
1863         diff = ""
1864         for editedFile in editedFiles:
1865             diff += p4_read_pipe(['diff', '-du',
1866                                   wildcard_encode(editedFile)])
1867
1868         # new file diff
1869         newdiff = ""
1870         for newFile in filesToAdd:
1871             newdiff += "==== new file ====\n"
1872             newdiff += "--- /dev/null\n"
1873             newdiff += "+++ %s\n" % newFile
1874
1875             is_link = os.path.islink(newFile)
1876             expect_link = newFile in symlinks
1877
1878             if is_link and expect_link:
1879                 newdiff += "+%s\n" % os.readlink(newFile)
1880             else:
1881                 f = open(newFile, "r")
1882                 for line in f.readlines():
1883                     newdiff += "+" + line
1884                 f.close()
1885
1886         return (diff + newdiff).replace('\r\n', '\n')
1887
1888     def applyCommit(self, id):
1889         """Apply one commit, return True if it succeeded."""
1890
1891         print("Applying", read_pipe(["git", "show", "-s",
1892                                      "--format=format:%h %s", id]))
1893
1894         (p4User, gitEmail) = self.p4UserForCommit(id)
1895
1896         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1897         filesToAdd = set()
1898         filesToChangeType = set()
1899         filesToDelete = set()
1900         editedFiles = set()
1901         pureRenameCopy = set()
1902         symlinks = set()
1903         filesToChangeExecBit = {}
1904         all_files = list()
1905
1906         for line in diff:
1907             diff = parseDiffTreeEntry(line)
1908             modifier = diff['status']
1909             path = diff['src']
1910             all_files.append(path)
1911
1912             if modifier == "M":
1913                 p4_edit(path)
1914                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1915                     filesToChangeExecBit[path] = diff['dst_mode']
1916                 editedFiles.add(path)
1917             elif modifier == "A":
1918                 filesToAdd.add(path)
1919                 filesToChangeExecBit[path] = diff['dst_mode']
1920                 if path in filesToDelete:
1921                     filesToDelete.remove(path)
1922
1923                 dst_mode = int(diff['dst_mode'], 8)
1924                 if dst_mode == 0o120000:
1925                     symlinks.add(path)
1926
1927             elif modifier == "D":
1928                 filesToDelete.add(path)
1929                 if path in filesToAdd:
1930                     filesToAdd.remove(path)
1931             elif modifier == "C":
1932                 src, dest = diff['src'], diff['dst']
1933                 all_files.append(dest)
1934                 p4_integrate(src, dest)
1935                 pureRenameCopy.add(dest)
1936                 if diff['src_sha1'] != diff['dst_sha1']:
1937                     p4_edit(dest)
1938                     pureRenameCopy.discard(dest)
1939                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1940                     p4_edit(dest)
1941                     pureRenameCopy.discard(dest)
1942                     filesToChangeExecBit[dest] = diff['dst_mode']
1943                 if self.isWindows:
1944                     # turn off read-only attribute
1945                     os.chmod(dest, stat.S_IWRITE)
1946                 os.unlink(dest)
1947                 editedFiles.add(dest)
1948             elif modifier == "R":
1949                 src, dest = diff['src'], diff['dst']
1950                 all_files.append(dest)
1951                 if self.p4HasMoveCommand:
1952                     p4_edit(src)        # src must be open before move
1953                     p4_move(src, dest)  # opens for (move/delete, move/add)
1954                 else:
1955                     p4_integrate(src, dest)
1956                     if diff['src_sha1'] != diff['dst_sha1']:
1957                         p4_edit(dest)
1958                     else:
1959                         pureRenameCopy.add(dest)
1960                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1961                     if not self.p4HasMoveCommand:
1962                         p4_edit(dest)   # with move: already open, writable
1963                     filesToChangeExecBit[dest] = diff['dst_mode']
1964                 if not self.p4HasMoveCommand:
1965                     if self.isWindows:
1966                         os.chmod(dest, stat.S_IWRITE)
1967                     os.unlink(dest)
1968                     filesToDelete.add(src)
1969                 editedFiles.add(dest)
1970             elif modifier == "T":
1971                 filesToChangeType.add(path)
1972             else:
1973                 die("unknown modifier %s for %s" % (modifier, path))
1974
1975         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1976         patchcmd = diffcmd + " | git apply "
1977         tryPatchCmd = patchcmd + "--check -"
1978         applyPatchCmd = patchcmd + "--check --apply -"
1979         patch_succeeded = True
1980
1981         if os.system(tryPatchCmd) != 0:
1982             fixed_rcs_keywords = False
1983             patch_succeeded = False
1984             print("Unfortunately applying the change failed!")
1985
1986             # Patch failed, maybe it's just RCS keyword woes. Look through
1987             # the patch to see if that's possible.
1988             if gitConfigBool("git-p4.attemptRCSCleanup"):
1989                 file = None
1990                 pattern = None
1991                 kwfiles = {}
1992                 for file in editedFiles | filesToDelete:
1993                     # did this file's delta contain RCS keywords?
1994                     pattern = p4_keywords_regexp_for_file(file)
1995
1996                     if pattern:
1997                         # this file is a possibility...look for RCS keywords.
1998                         regexp = re.compile(pattern, re.VERBOSE)
1999                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
2000                             if regexp.search(line):
2001                                 if verbose:
2002                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
2003                                 kwfiles[file] = pattern
2004                                 break
2005
2006                 for file in kwfiles:
2007                     if verbose:
2008                         print("zapping %s with %s" % (line,pattern))
2009                     # File is being deleted, so not open in p4.  Must
2010                     # disable the read-only bit on windows.
2011                     if self.isWindows and file not in editedFiles:
2012                         os.chmod(file, stat.S_IWRITE)
2013                     self.patchRCSKeywords(file, kwfiles[file])
2014                     fixed_rcs_keywords = True
2015
2016             if fixed_rcs_keywords:
2017                 print("Retrying the patch with RCS keywords cleaned up")
2018                 if os.system(tryPatchCmd) == 0:
2019                     patch_succeeded = True
2020
2021         if not patch_succeeded:
2022             for f in editedFiles:
2023                 p4_revert(f)
2024             return False
2025
2026         #
2027         # Apply the patch for real, and do add/delete/+x handling.
2028         #
2029         system(applyPatchCmd)
2030
2031         for f in filesToChangeType:
2032             p4_edit(f, "-t", "auto")
2033         for f in filesToAdd:
2034             p4_add(f)
2035         for f in filesToDelete:
2036             p4_revert(f)
2037             p4_delete(f)
2038
2039         # Set/clear executable bits
2040         for f in filesToChangeExecBit.keys():
2041             mode = filesToChangeExecBit[f]
2042             setP4ExecBit(f, mode)
2043
2044         update_shelve = 0
2045         if len(self.update_shelve) > 0:
2046             update_shelve = self.update_shelve.pop(0)
2047             p4_reopen_in_change(update_shelve, all_files)
2048
2049         #
2050         # Build p4 change description, starting with the contents
2051         # of the git commit message.
2052         #
2053         logMessage = extractLogMessageFromGitCommit(id)
2054         logMessage = logMessage.strip()
2055         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
2056
2057         template = self.prepareSubmitTemplate(update_shelve)
2058         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
2059
2060         if self.preserveUser:
2061            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
2062
2063         if self.checkAuthorship and not self.p4UserIsMe(p4User):
2064             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2065             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2066             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2067
2068         separatorLine = "######## everything below this line is just the diff #######\n"
2069         if not self.prepare_p4_only:
2070             submitTemplate += separatorLine
2071             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2072
2073         (handle, fileName) = tempfile.mkstemp()
2074         tmpFile = os.fdopen(handle, "w+b")
2075         if self.isWindows:
2076             submitTemplate = submitTemplate.replace("\n", "\r\n")
2077         tmpFile.write(encode_text_stream(submitTemplate))
2078         tmpFile.close()
2079
2080         if self.prepare_p4_only:
2081             #
2082             # Leave the p4 tree prepared, and the submit template around
2083             # and let the user decide what to do next
2084             #
2085             print()
2086             print("P4 workspace prepared for submission.")
2087             print("To submit or revert, go to client workspace")
2088             print("  " + self.clientPath)
2089             print()
2090             print("To submit, use \"p4 submit\" to write a new description,")
2091             print("or \"p4 submit -i <%s\" to use the one prepared by" \
2092                   " \"git p4\"." % fileName)
2093             print("You can delete the file \"%s\" when finished." % fileName)
2094
2095             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2096                 print("To preserve change ownership by user %s, you must\n" \
2097                       "do \"p4 change -f <change>\" after submitting and\n" \
2098                       "edit the User field.")
2099             if pureRenameCopy:
2100                 print("After submitting, renamed files must be re-synced.")
2101                 print("Invoke \"p4 sync -f\" on each of these files:")
2102                 for f in pureRenameCopy:
2103                     print("  " + f)
2104
2105             print()
2106             print("To revert the changes, use \"p4 revert ...\", and delete")
2107             print("the submit template file \"%s\"" % fileName)
2108             if filesToAdd:
2109                 print("Since the commit adds new files, they must be deleted:")
2110                 for f in filesToAdd:
2111                     print("  " + f)
2112             print()
2113             return True
2114
2115         #
2116         # Let the user edit the change description, then submit it.
2117         #
2118         submitted = False
2119
2120         try:
2121             if self.edit_template(fileName):
2122                 # read the edited message and submit
2123                 tmpFile = open(fileName, "rb")
2124                 message = decode_text_stream(tmpFile.read())
2125                 tmpFile.close()
2126                 if self.isWindows:
2127                     message = message.replace("\r\n", "\n")
2128                 submitTemplate = message[:message.index(separatorLine)]
2129
2130                 if update_shelve:
2131                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2132                 elif self.shelve:
2133                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2134                 else:
2135                     p4_write_pipe(['submit', '-i'], submitTemplate)
2136                     # The rename/copy happened by applying a patch that created a
2137                     # new file.  This leaves it writable, which confuses p4.
2138                     for f in pureRenameCopy:
2139                         p4_sync(f, "-f")
2140
2141                 if self.preserveUser:
2142                     if p4User:
2143                         # Get last changelist number. Cannot easily get it from
2144                         # the submit command output as the output is
2145                         # unmarshalled.
2146                         changelist = self.lastP4Changelist()
2147                         self.modifyChangelistUser(changelist, p4User)
2148
2149                 submitted = True
2150
2151         finally:
2152             # skip this patch
2153             if not submitted or self.shelve:
2154                 if self.shelve:
2155                     print ("Reverting shelved files.")
2156                 else:
2157                     print ("Submission cancelled, undoing p4 changes.")
2158                 for f in editedFiles | filesToDelete:
2159                     p4_revert(f)
2160                 for f in filesToAdd:
2161                     p4_revert(f)
2162                     os.remove(f)
2163
2164         os.remove(fileName)
2165         return submitted
2166
2167     # Export git tags as p4 labels. Create a p4 label and then tag
2168     # with that.
2169     def exportGitTags(self, gitTags):
2170         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2171         if len(validLabelRegexp) == 0:
2172             validLabelRegexp = defaultLabelRegexp
2173         m = re.compile(validLabelRegexp)
2174
2175         for name in gitTags:
2176
2177             if not m.match(name):
2178                 if verbose:
2179                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2180                 continue
2181
2182             # Get the p4 commit this corresponds to
2183             logMessage = extractLogMessageFromGitCommit(name)
2184             values = extractSettingsGitLog(logMessage)
2185
2186             if 'change' not in values:
2187                 # a tag pointing to something not sent to p4; ignore
2188                 if verbose:
2189                     print("git tag %s does not give a p4 commit" % name)
2190                 continue
2191             else:
2192                 changelist = values['change']
2193
2194             # Get the tag details.
2195             inHeader = True
2196             isAnnotated = False
2197             body = []
2198             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2199                 l = l.strip()
2200                 if inHeader:
2201                     if re.match(r'tag\s+', l):
2202                         isAnnotated = True
2203                     elif re.match(r'\s*$', l):
2204                         inHeader = False
2205                         continue
2206                 else:
2207                     body.append(l)
2208
2209             if not isAnnotated:
2210                 body = ["lightweight tag imported by git p4\n"]
2211
2212             # Create the label - use the same view as the client spec we are using
2213             clientSpec = getClientSpec()
2214
2215             labelTemplate  = "Label: %s\n" % name
2216             labelTemplate += "Description:\n"
2217             for b in body:
2218                 labelTemplate += "\t" + b + "\n"
2219             labelTemplate += "View:\n"
2220             for depot_side in clientSpec.mappings:
2221                 labelTemplate += "\t%s\n" % depot_side
2222
2223             if self.dry_run:
2224                 print("Would create p4 label %s for tag" % name)
2225             elif self.prepare_p4_only:
2226                 print("Not creating p4 label %s for tag due to option" \
2227                       " --prepare-p4-only" % name)
2228             else:
2229                 p4_write_pipe(["label", "-i"], labelTemplate)
2230
2231                 # Use the label
2232                 p4_system(["tag", "-l", name] +
2233                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2234
2235                 if verbose:
2236                     print("created p4 label for tag %s" % name)
2237
2238     def run(self, args):
2239         if len(args) == 0:
2240             self.master = currentGitBranch()
2241         elif len(args) == 1:
2242             self.master = args[0]
2243             if not branchExists(self.master):
2244                 die("Branch %s does not exist" % self.master)
2245         else:
2246             return False
2247
2248         for i in self.update_shelve:
2249             if i <= 0:
2250                 sys.exit("invalid changelist %d" % i)
2251
2252         if self.master:
2253             allowSubmit = gitConfig("git-p4.allowSubmit")
2254             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2255                 die("%s is not in git-p4.allowSubmit" % self.master)
2256
2257         [upstream, settings] = findUpstreamBranchPoint()
2258         self.depotPath = settings['depot-paths'][0]
2259         if len(self.origin) == 0:
2260             self.origin = upstream
2261
2262         if len(self.update_shelve) > 0:
2263             self.shelve = True
2264
2265         if self.preserveUser:
2266             if not self.canChangeChangelists():
2267                 die("Cannot preserve user names without p4 super-user or admin permissions")
2268
2269         # if not set from the command line, try the config file
2270         if self.conflict_behavior is None:
2271             val = gitConfig("git-p4.conflict")
2272             if val:
2273                 if val not in self.conflict_behavior_choices:
2274                     die("Invalid value '%s' for config git-p4.conflict" % val)
2275             else:
2276                 val = "ask"
2277             self.conflict_behavior = val
2278
2279         if self.verbose:
2280             print("Origin branch is " + self.origin)
2281
2282         if len(self.depotPath) == 0:
2283             print("Internal error: cannot locate perforce depot path from existing branches")
2284             sys.exit(128)
2285
2286         self.useClientSpec = False
2287         if gitConfigBool("git-p4.useclientspec"):
2288             self.useClientSpec = True
2289         if self.useClientSpec:
2290             self.clientSpecDirs = getClientSpec()
2291
2292         # Check for the existence of P4 branches
2293         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2294
2295         if self.useClientSpec and not branchesDetected:
2296             # all files are relative to the client spec
2297             self.clientPath = getClientRoot()
2298         else:
2299             self.clientPath = p4Where(self.depotPath)
2300
2301         if self.clientPath == "":
2302             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2303
2304         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2305         self.oldWorkingDirectory = os.getcwd()
2306
2307         # ensure the clientPath exists
2308         new_client_dir = False
2309         if not os.path.exists(self.clientPath):
2310             new_client_dir = True
2311             os.makedirs(self.clientPath)
2312
2313         chdir(self.clientPath, is_client_path=True)
2314         if self.dry_run:
2315             print("Would synchronize p4 checkout in %s" % self.clientPath)
2316         else:
2317             print("Synchronizing p4 checkout...")
2318             if new_client_dir:
2319                 # old one was destroyed, and maybe nobody told p4
2320                 p4_sync("...", "-f")
2321             else:
2322                 p4_sync("...")
2323         self.check()
2324
2325         commits = []
2326         if self.master:
2327             committish = self.master
2328         else:
2329             committish = 'HEAD'
2330
2331         if self.commit != "":
2332             if self.commit.find("..") != -1:
2333                 limits_ish = self.commit.split("..")
2334                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2335                     commits.append(line.strip())
2336                 commits.reverse()
2337             else:
2338                 commits.append(self.commit)
2339         else:
2340             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2341                 commits.append(line.strip())
2342             commits.reverse()
2343
2344         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2345             self.checkAuthorship = False
2346         else:
2347             self.checkAuthorship = True
2348
2349         if self.preserveUser:
2350             self.checkValidP4Users(commits)
2351
2352         #
2353         # Build up a set of options to be passed to diff when
2354         # submitting each commit to p4.
2355         #
2356         if self.detectRenames:
2357             # command-line -M arg
2358             self.diffOpts = "-M"
2359         else:
2360             # If not explicitly set check the config variable
2361             detectRenames = gitConfig("git-p4.detectRenames")
2362
2363             if detectRenames.lower() == "false" or detectRenames == "":
2364                 self.diffOpts = ""
2365             elif detectRenames.lower() == "true":
2366                 self.diffOpts = "-M"
2367             else:
2368                 self.diffOpts = "-M%s" % detectRenames
2369
2370         # no command-line arg for -C or --find-copies-harder, just
2371         # config variables
2372         detectCopies = gitConfig("git-p4.detectCopies")
2373         if detectCopies.lower() == "false" or detectCopies == "":
2374             pass
2375         elif detectCopies.lower() == "true":
2376             self.diffOpts += " -C"
2377         else:
2378             self.diffOpts += " -C%s" % detectCopies
2379
2380         if gitConfigBool("git-p4.detectCopiesHarder"):
2381             self.diffOpts += " --find-copies-harder"
2382
2383         num_shelves = len(self.update_shelve)
2384         if num_shelves > 0 and num_shelves != len(commits):
2385             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2386                      (len(commits), num_shelves))
2387
2388         hooks_path = gitConfig("core.hooksPath")
2389         if len(hooks_path) <= 0:
2390             hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2391
2392         hook_file = os.path.join(hooks_path, "p4-pre-submit")
2393         if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2394             sys.exit(1)
2395
2396         #
2397         # Apply the commits, one at a time.  On failure, ask if should
2398         # continue to try the rest of the patches, or quit.
2399         #
2400         if self.dry_run:
2401             print("Would apply")
2402         applied = []
2403         last = len(commits) - 1
2404         for i, commit in enumerate(commits):
2405             if self.dry_run:
2406                 print(" ", read_pipe(["git", "show", "-s",
2407                                       "--format=format:%h %s", commit]))
2408                 ok = True
2409             else:
2410                 ok = self.applyCommit(commit)
2411             if ok:
2412                 applied.append(commit)
2413             else:
2414                 if self.prepare_p4_only and i < last:
2415                     print("Processing only the first commit due to option" \
2416                           " --prepare-p4-only")
2417                     break
2418                 if i < last:
2419                     # prompt for what to do, or use the option/variable
2420                     if self.conflict_behavior == "ask":
2421                         print("What do you want to do?")
2422                         response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2423                     elif self.conflict_behavior == "skip":
2424                         response = "s"
2425                     elif self.conflict_behavior == "quit":
2426                         response = "q"
2427                     else:
2428                         die("Unknown conflict_behavior '%s'" %
2429                             self.conflict_behavior)
2430
2431                     if response == "s":
2432                         print("Skipping this commit, but applying the rest")
2433                     if response == "q":
2434                         print("Quitting")
2435                         break
2436
2437         chdir(self.oldWorkingDirectory)
2438         shelved_applied = "shelved" if self.shelve else "applied"
2439         if self.dry_run:
2440             pass
2441         elif self.prepare_p4_only:
2442             pass
2443         elif len(commits) == len(applied):
2444             print("All commits {0}!".format(shelved_applied))
2445
2446             sync = P4Sync()
2447             if self.branch:
2448                 sync.branch = self.branch
2449             if self.disable_p4sync:
2450                 sync.sync_origin_only()
2451             else:
2452                 sync.run([])
2453
2454                 if not self.disable_rebase:
2455                     rebase = P4Rebase()
2456                     rebase.rebase()
2457
2458         else:
2459             if len(applied) == 0:
2460                 print("No commits {0}.".format(shelved_applied))
2461             else:
2462                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2463                 for c in commits:
2464                     if c in applied:
2465                         star = "*"
2466                     else:
2467                         star = " "
2468                     print(star, read_pipe(["git", "show", "-s",
2469                                            "--format=format:%h %s",  c]))
2470                 print("You will have to do 'git p4 sync' and rebase.")
2471
2472         if gitConfigBool("git-p4.exportLabels"):
2473             self.exportLabels = True
2474
2475         if self.exportLabels:
2476             p4Labels = getP4Labels(self.depotPath)
2477             gitTags = getGitTags()
2478
2479             missingGitTags = gitTags - p4Labels
2480             self.exportGitTags(missingGitTags)
2481
2482         # exit with error unless everything applied perfectly
2483         if len(commits) != len(applied):
2484                 sys.exit(1)
2485
2486         return True
2487
2488 class View(object):
2489     """Represent a p4 view ("p4 help views"), and map files in a
2490        repo according to the view."""
2491
2492     def __init__(self, client_name):
2493         self.mappings = []
2494         self.client_prefix = "//%s/" % client_name
2495         # cache results of "p4 where" to lookup client file locations
2496         self.client_spec_path_cache = {}
2497
2498     def append(self, view_line):
2499         """Parse a view line, splitting it into depot and client
2500            sides.  Append to self.mappings, preserving order.  This
2501            is only needed for tag creation."""
2502
2503         # Split the view line into exactly two words.  P4 enforces
2504         # structure on these lines that simplifies this quite a bit.
2505         #
2506         # Either or both words may be double-quoted.
2507         # Single quotes do not matter.
2508         # Double-quote marks cannot occur inside the words.
2509         # A + or - prefix is also inside the quotes.
2510         # There are no quotes unless they contain a space.
2511         # The line is already white-space stripped.
2512         # The two words are separated by a single space.
2513         #
2514         if view_line[0] == '"':
2515             # First word is double quoted.  Find its end.
2516             close_quote_index = view_line.find('"', 1)
2517             if close_quote_index <= 0:
2518                 die("No first-word closing quote found: %s" % view_line)
2519             depot_side = view_line[1:close_quote_index]
2520             # skip closing quote and space
2521             rhs_index = close_quote_index + 1 + 1
2522         else:
2523             space_index = view_line.find(" ")
2524             if space_index <= 0:
2525                 die("No word-splitting space found: %s" % view_line)
2526             depot_side = view_line[0:space_index]
2527             rhs_index = space_index + 1
2528
2529         # prefix + means overlay on previous mapping
2530         if depot_side.startswith("+"):
2531             depot_side = depot_side[1:]
2532
2533         # prefix - means exclude this path, leave out of mappings
2534         exclude = False
2535         if depot_side.startswith("-"):
2536             exclude = True
2537             depot_side = depot_side[1:]
2538
2539         if not exclude:
2540             self.mappings.append(depot_side)
2541
2542     def convert_client_path(self, clientFile):
2543         # chop off //client/ part to make it relative
2544         if not decode_path(clientFile).startswith(self.client_prefix):
2545             die("No prefix '%s' on clientFile '%s'" %
2546                 (self.client_prefix, clientFile))
2547         return clientFile[len(self.client_prefix):]
2548
2549     def update_client_spec_path_cache(self, files):
2550         """ Caching file paths by "p4 where" batch query """
2551
2552         # List depot file paths exclude that already cached
2553         fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
2554
2555         if len(fileArgs) == 0:
2556             return  # All files in cache
2557
2558         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2559         for res in where_result:
2560             if "code" in res and res["code"] == "error":
2561                 # assume error is "... file(s) not in client view"
2562                 continue
2563             if "clientFile" not in res:
2564                 die("No clientFile in 'p4 where' output")
2565             if "unmap" in res:
2566                 # it will list all of them, but only one not unmap-ped
2567                 continue
2568             depot_path = decode_path(res['depotFile'])
2569             if gitConfigBool("core.ignorecase"):
2570                 depot_path = depot_path.lower()
2571             self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
2572
2573         # not found files or unmap files set to ""
2574         for depotFile in fileArgs:
2575             depotFile = decode_path(depotFile)
2576             if gitConfigBool("core.ignorecase"):
2577                 depotFile = depotFile.lower()
2578             if depotFile not in self.client_spec_path_cache:
2579                 self.client_spec_path_cache[depotFile] = b''
2580
2581     def map_in_client(self, depot_path):
2582         """Return the relative location in the client where this
2583            depot file should live.  Returns "" if the file should
2584            not be mapped in the client."""
2585
2586         if gitConfigBool("core.ignorecase"):
2587             depot_path = depot_path.lower()
2588
2589         if depot_path in self.client_spec_path_cache:
2590             return self.client_spec_path_cache[depot_path]
2591
2592         die( "Error: %s is not found in client spec path" % depot_path )
2593         return ""
2594
2595 def cloneExcludeCallback(option, opt_str, value, parser):
2596     # prepend "/" because the first "/" was consumed as part of the option itself.
2597     # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2598     parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2599
2600 class P4Sync(Command, P4UserMap):
2601
2602     def __init__(self):
2603         Command.__init__(self)
2604         P4UserMap.__init__(self)
2605         self.options = [
2606                 optparse.make_option("--branch", dest="branch"),
2607                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2608                 optparse.make_option("--changesfile", dest="changesFile"),
2609                 optparse.make_option("--silent", dest="silent", action="store_true"),
2610                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2611                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2612                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2613                                      help="Import into refs/heads/ , not refs/remotes"),
2614                 optparse.make_option("--max-changes", dest="maxChanges",
2615                                      help="Maximum number of changes to import"),
2616                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2617                                      help="Internal block size to use when iteratively calling p4 changes"),
2618                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2619                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2620                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2621                                      help="Only sync files that are included in the Perforce Client Spec"),
2622                 optparse.make_option("-/", dest="cloneExclude",
2623                                      action="callback", callback=cloneExcludeCallback, type="string",
2624                                      help="exclude depot path"),
2625         ]
2626         self.description = """Imports from Perforce into a git repository.\n
2627     example:
2628     //depot/my/project/ -- to import the current head
2629     //depot/my/project/@all -- to import everything
2630     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2631
2632     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2633
2634         self.usage += " //depot/path[@revRange]"
2635         self.silent = False
2636         self.createdBranches = set()
2637         self.committedChanges = set()
2638         self.branch = ""
2639         self.detectBranches = False
2640         self.detectLabels = False
2641         self.importLabels = False
2642         self.changesFile = ""
2643         self.syncWithOrigin = True
2644         self.importIntoRemotes = True
2645         self.maxChanges = ""
2646         self.changes_block_size = None
2647         self.keepRepoPath = False
2648         self.depotPaths = None
2649         self.p4BranchesInGit = []
2650         self.cloneExclude = []
2651         self.useClientSpec = False
2652         self.useClientSpec_from_options = False
2653         self.clientSpecDirs = None
2654         self.tempBranches = []
2655         self.tempBranchLocation = "refs/git-p4-tmp"
2656         self.largeFileSystem = None
2657         self.suppress_meta_comment = False
2658
2659         if gitConfig('git-p4.largeFileSystem'):
2660             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2661             self.largeFileSystem = largeFileSystemConstructor(
2662                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2663             )
2664
2665         if gitConfig("git-p4.syncFromOrigin") == "false":
2666             self.syncWithOrigin = False
2667
2668         self.depotPaths = []
2669         self.changeRange = ""
2670         self.previousDepotPaths = []
2671         self.hasOrigin = False
2672
2673         # map from branch depot path to parent branch
2674         self.knownBranches = {}
2675         self.initialParents = {}
2676
2677         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2678         self.labels = {}
2679
2680     # Force a checkpoint in fast-import and wait for it to finish
2681     def checkpoint(self):
2682         self.gitStream.write("checkpoint\n\n")
2683         self.gitStream.write("progress checkpoint\n\n")
2684         self.gitStream.flush()
2685         out = self.gitOutput.readline()
2686         if self.verbose:
2687             print("checkpoint finished: " + out)
2688
2689     def isPathWanted(self, path):
2690         for p in self.cloneExclude:
2691             if p.endswith("/"):
2692                 if p4PathStartsWith(path, p):
2693                     return False
2694             # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2695             elif path.lower() == p.lower():
2696                 return False
2697         for p in self.depotPaths:
2698             if p4PathStartsWith(path, decode_path(p)):
2699                 return True
2700         return False
2701
2702     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2703         files = []
2704         fnum = 0
2705         while "depotFile%s" % fnum in commit:
2706             path =  commit["depotFile%s" % fnum]
2707             found = self.isPathWanted(decode_path(path))
2708             if not found:
2709                 fnum = fnum + 1
2710                 continue
2711
2712             file = {}
2713             file["path"] = path
2714             file["rev"] = commit["rev%s" % fnum]
2715             file["action"] = commit["action%s" % fnum]
2716             file["type"] = commit["type%s" % fnum]
2717             if shelved:
2718                 file["shelved_cl"] = int(shelved_cl)
2719             files.append(file)
2720             fnum = fnum + 1
2721         return files
2722
2723     def extractJobsFromCommit(self, commit):
2724         jobs = []
2725         jnum = 0
2726         while "job%s" % jnum in commit:
2727             job = commit["job%s" % jnum]
2728             jobs.append(job)
2729             jnum = jnum + 1
2730         return jobs
2731
2732     def stripRepoPath(self, path, prefixes):
2733         """When streaming files, this is called to map a p4 depot path
2734            to where it should go in git.  The prefixes are either
2735            self.depotPaths, or self.branchPrefixes in the case of
2736            branch detection."""
2737
2738         if self.useClientSpec:
2739             # branch detection moves files up a level (the branch name)
2740             # from what client spec interpretation gives
2741             path = decode_path(self.clientSpecDirs.map_in_client(path))
2742             if self.detectBranches:
2743                 for b in self.knownBranches:
2744                     if p4PathStartsWith(path, b + "/"):
2745                         path = path[len(b)+1:]
2746
2747         elif self.keepRepoPath:
2748             # Preserve everything in relative path name except leading
2749             # //depot/; just look at first prefix as they all should
2750             # be in the same depot.
2751             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2752             if p4PathStartsWith(path, depot):
2753                 path = path[len(depot):]
2754
2755         else:
2756             for p in prefixes:
2757                 if p4PathStartsWith(path, p):
2758                     path = path[len(p):]
2759                     break
2760
2761         path = wildcard_decode(path)
2762         return path
2763
2764     def splitFilesIntoBranches(self, commit):
2765         """Look at each depotFile in the commit to figure out to what
2766            branch it belongs."""
2767
2768         if self.clientSpecDirs:
2769             files = self.extractFilesFromCommit(commit)
2770             self.clientSpecDirs.update_client_spec_path_cache(files)
2771
2772         branches = {}
2773         fnum = 0
2774         while "depotFile%s" % fnum in commit:
2775             raw_path = commit["depotFile%s" % fnum]
2776             path = decode_path(raw_path)
2777             found = self.isPathWanted(path)
2778             if not found:
2779                 fnum = fnum + 1
2780                 continue
2781
2782             file = {}
2783             file["path"] = raw_path
2784             file["rev"] = commit["rev%s" % fnum]
2785             file["action"] = commit["action%s" % fnum]
2786             file["type"] = commit["type%s" % fnum]
2787             fnum = fnum + 1
2788
2789             # start with the full relative path where this file would
2790             # go in a p4 client
2791             if self.useClientSpec:
2792                 relPath = decode_path(self.clientSpecDirs.map_in_client(path))
2793             else:
2794                 relPath = self.stripRepoPath(path, self.depotPaths)
2795
2796             for branch in self.knownBranches.keys():
2797                 # add a trailing slash so that a commit into qt/4.2foo
2798                 # doesn't end up in qt/4.2, e.g.
2799                 if p4PathStartsWith(relPath, branch + "/"):
2800                     if branch not in branches:
2801                         branches[branch] = []
2802                     branches[branch].append(file)
2803                     break
2804
2805         return branches
2806
2807     def writeToGitStream(self, gitMode, relPath, contents):
2808         self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
2809         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2810         for d in contents:
2811             self.gitStream.write(d)
2812         self.gitStream.write('\n')
2813
2814     def encodeWithUTF8(self, path):
2815         try:
2816             path.decode('ascii')
2817         except:
2818             encoding = 'utf8'
2819             if gitConfig('git-p4.pathEncoding'):
2820                 encoding = gitConfig('git-p4.pathEncoding')
2821             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2822             if self.verbose:
2823                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2824         return path
2825
2826     # output one file from the P4 stream
2827     # - helper for streamP4Files
2828
2829     def streamOneP4File(self, file, contents):
2830         file_path = file['depotFile']
2831         relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
2832
2833         if verbose:
2834             if 'fileSize' in self.stream_file:
2835                 size = int(self.stream_file['fileSize'])
2836             else:
2837                 size = 0 # deleted files don't get a fileSize apparently
2838             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file_path, relPath, size/1024/1024))
2839             sys.stdout.flush()
2840
2841         (type_base, type_mods) = split_p4_type(file["type"])
2842
2843         git_mode = "100644"
2844         if "x" in type_mods:
2845             git_mode = "100755"
2846         if type_base == "symlink":
2847             git_mode = "120000"
2848             # p4 print on a symlink sometimes contains "target\n";
2849             # if it does, remove the newline
2850             data = ''.join(decode_text_stream(c) for c in contents)
2851             if not data:
2852                 # Some version of p4 allowed creating a symlink that pointed
2853                 # to nothing.  This causes p4 errors when checking out such
2854                 # a change, and errors here too.  Work around it by ignoring
2855                 # the bad symlink; hopefully a future change fixes it.
2856                 print("\nIgnoring empty symlink in %s" % file_path)
2857                 return
2858             elif data[-1] == '\n':
2859                 contents = [data[:-1]]
2860             else:
2861                 contents = [data]
2862
2863         if type_base == "utf16":
2864             # p4 delivers different text in the python output to -G
2865             # than it does when using "print -o", or normal p4 client
2866             # operations.  utf16 is converted to ascii or utf8, perhaps.
2867             # But ascii text saved as -t utf16 is completely mangled.
2868             # Invoke print -o to get the real contents.
2869             #
2870             # On windows, the newlines will always be mangled by print, so put
2871             # them back too.  This is not needed to the cygwin windows version,
2872             # just the native "NT" type.
2873             #
2874             try:
2875                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
2876             except Exception as e:
2877                 if 'Translation of file content failed' in str(e):
2878                     type_base = 'binary'
2879                 else:
2880                     raise e
2881             else:
2882                 if p4_version_string().find('/NT') >= 0:
2883                     text = text.replace(b'\r\n', b'\n')
2884                 contents = [ text ]
2885
2886         if type_base == "apple":
2887             # Apple filetype files will be streamed as a concatenation of
2888             # its appledouble header and the contents.  This is useless
2889             # on both macs and non-macs.  If using "print -q -o xx", it
2890             # will create "xx" with the data, and "%xx" with the header.
2891             # This is also not very useful.
2892             #
2893             # Ideally, someday, this script can learn how to generate
2894             # appledouble files directly and import those to git, but
2895             # non-mac machines can never find a use for apple filetype.
2896             print("\nIgnoring apple filetype file %s" % file['depotFile'])
2897             return
2898
2899         # Note that we do not try to de-mangle keywords on utf16 files,
2900         # even though in theory somebody may want that.
2901         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2902         if pattern:
2903             regexp = re.compile(pattern, re.VERBOSE)
2904             text = ''.join(decode_text_stream(c) for c in contents)
2905             text = regexp.sub(r'$\1$', text)
2906             contents = [ text ]
2907
2908         if self.largeFileSystem:
2909             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2910
2911         self.writeToGitStream(git_mode, relPath, contents)
2912
2913     def streamOneP4Deletion(self, file):
2914         relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
2915         if verbose:
2916             sys.stdout.write("delete %s\n" % relPath)
2917             sys.stdout.flush()
2918         self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
2919
2920         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2921             self.largeFileSystem.removeLargeFile(relPath)
2922
2923     # handle another chunk of streaming data
2924     def streamP4FilesCb(self, marshalled):
2925
2926         # catch p4 errors and complain
2927         err = None
2928         if "code" in marshalled:
2929             if marshalled["code"] == "error":
2930                 if "data" in marshalled:
2931                     err = marshalled["data"].rstrip()
2932
2933         if not err and 'fileSize' in self.stream_file:
2934             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2935             if required_bytes > 0:
2936                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2937                     os.getcwd(), required_bytes/1024/1024
2938                 )
2939
2940         if err:
2941             f = None
2942             if self.stream_have_file_info:
2943                 if "depotFile" in self.stream_file:
2944                     f = self.stream_file["depotFile"]
2945             # force a failure in fast-import, else an empty
2946             # commit will be made
2947             self.gitStream.write("\n")
2948             self.gitStream.write("die-now\n")
2949             self.gitStream.close()
2950             # ignore errors, but make sure it exits first
2951             self.importProcess.wait()
2952             if f:
2953                 die("Error from p4 print for %s: %s" % (f, err))
2954             else:
2955                 die("Error from p4 print: %s" % err)
2956
2957         if 'depotFile' in marshalled and self.stream_have_file_info:
2958             # start of a new file - output the old one first
2959             self.streamOneP4File(self.stream_file, self.stream_contents)
2960             self.stream_file = {}
2961             self.stream_contents = []
2962             self.stream_have_file_info = False
2963
2964         # pick up the new file information... for the
2965         # 'data' field we need to append to our array
2966         for k in marshalled.keys():
2967             if k == 'data':
2968                 if 'streamContentSize' not in self.stream_file:
2969                     self.stream_file['streamContentSize'] = 0
2970                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2971                 self.stream_contents.append(marshalled['data'])
2972             else:
2973                 self.stream_file[k] = marshalled[k]
2974
2975         if (verbose and
2976             'streamContentSize' in self.stream_file and
2977             'fileSize' in self.stream_file and
2978             'depotFile' in self.stream_file):
2979             size = int(self.stream_file["fileSize"])
2980             if size > 0:
2981                 progress = 100*self.stream_file['streamContentSize']/size
2982                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2983                 sys.stdout.flush()
2984
2985         self.stream_have_file_info = True
2986
2987     # Stream directly from "p4 files" into "git fast-import"
2988     def streamP4Files(self, files):
2989         filesForCommit = []
2990         filesToRead = []
2991         filesToDelete = []
2992
2993         for f in files:
2994             filesForCommit.append(f)
2995             if f['action'] in self.delete_actions:
2996                 filesToDelete.append(f)
2997             else:
2998                 filesToRead.append(f)
2999
3000         # deleted files...
3001         for f in filesToDelete:
3002             self.streamOneP4Deletion(f)
3003
3004         if len(filesToRead) > 0:
3005             self.stream_file = {}
3006             self.stream_contents = []
3007             self.stream_have_file_info = False
3008
3009             # curry self argument
3010             def streamP4FilesCbSelf(entry):
3011                 self.streamP4FilesCb(entry)
3012
3013             fileArgs = []
3014             for f in filesToRead:
3015                 if 'shelved_cl' in f:
3016                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
3017                     # the contents
3018                     fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
3019                 else:
3020                     fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
3021
3022                 fileArgs.append(fileArg)
3023
3024             p4CmdList(["-x", "-", "print"],
3025                       stdin=fileArgs,
3026                       cb=streamP4FilesCbSelf)
3027
3028             # do the last chunk
3029             if 'depotFile' in self.stream_file:
3030                 self.streamOneP4File(self.stream_file, self.stream_contents)
3031
3032     def make_email(self, userid):
3033         if userid in self.users:
3034             return self.users[userid]
3035         else:
3036             return "%s <a@b>" % userid
3037
3038     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
3039         """ Stream a p4 tag.
3040         commit is either a git commit, or a fast-import mark, ":<p4commit>"
3041         """
3042
3043         if verbose:
3044             print("writing tag %s for commit %s" % (labelName, commit))
3045         gitStream.write("tag %s\n" % labelName)
3046         gitStream.write("from %s\n" % commit)
3047
3048         if 'Owner' in labelDetails:
3049             owner = labelDetails["Owner"]
3050         else:
3051             owner = None
3052
3053         # Try to use the owner of the p4 label, or failing that,
3054         # the current p4 user id.
3055         if owner:
3056             email = self.make_email(owner)
3057         else:
3058             email = self.make_email(self.p4UserId())
3059         tagger = "%s %s %s" % (email, epoch, self.tz)
3060
3061         gitStream.write("tagger %s\n" % tagger)
3062
3063         print("labelDetails=",labelDetails)
3064         if 'Description' in labelDetails:
3065             description = labelDetails['Description']
3066         else:
3067             description = 'Label from git p4'
3068
3069         gitStream.write("data %d\n" % len(description))
3070         gitStream.write(description)
3071         gitStream.write("\n")
3072
3073     def inClientSpec(self, path):
3074         if not self.clientSpecDirs:
3075             return True
3076         inClientSpec = self.clientSpecDirs.map_in_client(path)
3077         if not inClientSpec and self.verbose:
3078             print('Ignoring file outside of client spec: {0}'.format(path))
3079         return inClientSpec
3080
3081     def hasBranchPrefix(self, path):
3082         if not self.branchPrefixes:
3083             return True
3084         hasPrefix = [p for p in self.branchPrefixes
3085                         if p4PathStartsWith(path, p)]
3086         if not hasPrefix and self.verbose:
3087             print('Ignoring file outside of prefix: {0}'.format(path))
3088         return hasPrefix
3089
3090     def commit(self, details, files, branch, parent = "", allow_empty=False):
3091         epoch = details["time"]
3092         author = details["user"]
3093         jobs = self.extractJobsFromCommit(details)
3094
3095         if self.verbose:
3096             print('commit into {0}'.format(branch))
3097
3098         if self.clientSpecDirs:
3099             self.clientSpecDirs.update_client_spec_path_cache(files)
3100
3101         files = [f for (f, path) in ((f, decode_path(f['path'])) for f in files)
3102             if self.inClientSpec(path) and self.hasBranchPrefix(path)]
3103
3104         if gitConfigBool('git-p4.keepEmptyCommits'):
3105             allow_empty = True
3106
3107         if not files and not allow_empty:
3108             print('Ignoring revision {0} as it would produce an empty commit.'
3109                 .format(details['change']))
3110             return
3111
3112         self.gitStream.write("commit %s\n" % branch)
3113         self.gitStream.write("mark :%s\n" % details["change"])
3114         self.committedChanges.add(int(details["change"]))
3115         committer = ""
3116         if author not in self.users:
3117             self.getUserMapFromPerforceServer()
3118         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3119
3120         self.gitStream.write("committer %s\n" % committer)
3121
3122         self.gitStream.write("data <<EOT\n")
3123         self.gitStream.write(details["desc"])
3124         if len(jobs) > 0:
3125             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3126
3127         if not self.suppress_meta_comment:
3128             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3129                                 (','.join(self.branchPrefixes), details["change"]))
3130             if len(details['options']) > 0:
3131                 self.gitStream.write(": options = %s" % details['options'])
3132             self.gitStream.write("]\n")
3133
3134         self.gitStream.write("EOT\n\n")
3135
3136         if len(parent) > 0:
3137             if self.verbose:
3138                 print("parent %s" % parent)
3139             self.gitStream.write("from %s\n" % parent)
3140
3141         self.streamP4Files(files)
3142         self.gitStream.write("\n")
3143
3144         change = int(details["change"])
3145
3146         if change in self.labels:
3147             label = self.labels[change]
3148             labelDetails = label[0]
3149             labelRevisions = label[1]
3150             if self.verbose:
3151                 print("Change %s is labelled %s" % (change, labelDetails))
3152
3153             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3154                                                 for p in self.branchPrefixes])
3155
3156             if len(files) == len(labelRevisions):
3157
3158                 cleanedFiles = {}
3159                 for info in files:
3160                     if info["action"] in self.delete_actions:
3161                         continue
3162                     cleanedFiles[info["depotFile"]] = info["rev"]
3163
3164                 if cleanedFiles == labelRevisions:
3165                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3166
3167                 else:
3168                     if not self.silent:
3169                         print("Tag %s does not match with change %s: files do not match."
3170                                % (labelDetails["label"], change))
3171
3172             else:
3173                 if not self.silent:
3174                     print("Tag %s does not match with change %s: file count is different."
3175                            % (labelDetails["label"], change))
3176
3177     # Build a dictionary of changelists and labels, for "detect-labels" option.
3178     def getLabels(self):
3179         self.labels = {}
3180
3181         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3182         if len(l) > 0 and not self.silent:
3183             print("Finding files belonging to labels in %s" % self.depotPaths)
3184
3185         for output in l:
3186             label = output["label"]
3187             revisions = {}
3188             newestChange = 0
3189             if self.verbose:
3190                 print("Querying files for label %s" % label)
3191             for file in p4CmdList(["files"] +
3192                                       ["%s...@%s" % (p, label)
3193                                           for p in self.depotPaths]):
3194                 revisions[file["depotFile"]] = file["rev"]
3195                 change = int(file["change"])
3196                 if change > newestChange:
3197                     newestChange = change
3198
3199             self.labels[newestChange] = [output, revisions]
3200
3201         if self.verbose:
3202             print("Label changes: %s" % self.labels.keys())
3203
3204     # Import p4 labels as git tags. A direct mapping does not
3205     # exist, so assume that if all the files are at the same revision
3206     # then we can use that, or it's something more complicated we should
3207     # just ignore.
3208     def importP4Labels(self, stream, p4Labels):
3209         if verbose:
3210             print("import p4 labels: " + ' '.join(p4Labels))
3211
3212         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3213         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3214         if len(validLabelRegexp) == 0:
3215             validLabelRegexp = defaultLabelRegexp
3216         m = re.compile(validLabelRegexp)
3217
3218         for name in p4Labels:
3219             commitFound = False
3220
3221             if not m.match(name):
3222                 if verbose:
3223                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3224                 continue
3225
3226             if name in ignoredP4Labels:
3227                 continue
3228
3229             labelDetails = p4CmdList(['label', "-o", name])[0]
3230
3231             # get the most recent changelist for each file in this label
3232             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3233                                 for p in self.depotPaths])
3234
3235             if 'change' in change:
3236                 # find the corresponding git commit; take the oldest commit
3237                 changelist = int(change['change'])
3238                 if changelist in self.committedChanges:
3239                     gitCommit = ":%d" % changelist       # use a fast-import mark
3240                     commitFound = True
3241                 else:
3242                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3243                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3244                     if len(gitCommit) == 0:
3245                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3246                     else:
3247                         commitFound = True
3248                         gitCommit = gitCommit.strip()
3249
3250                 if commitFound:
3251                     # Convert from p4 time format
3252                     try:
3253                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3254                     except ValueError:
3255                         print("Could not convert label time %s" % labelDetails['Update'])
3256                         tmwhen = 1
3257
3258                     when = int(time.mktime(tmwhen))
3259                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3260                     if verbose:
3261                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3262             else:
3263                 if verbose:
3264                     print("Label %s has no changelists - possibly deleted?" % name)
3265
3266             if not commitFound:
3267                 # We can't import this label; don't try again as it will get very
3268                 # expensive repeatedly fetching all the files for labels that will
3269                 # never be imported. If the label is moved in the future, the
3270                 # ignore will need to be removed manually.
3271                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3272
3273     def guessProjectName(self):
3274         for p in self.depotPaths:
3275             if p.endswith("/"):
3276                 p = p[:-1]
3277             p = p[p.strip().rfind("/") + 1:]
3278             if not p.endswith("/"):
3279                p += "/"
3280             return p
3281
3282     def getBranchMapping(self):
3283         lostAndFoundBranches = set()
3284
3285         user = gitConfig("git-p4.branchUser")
3286         if len(user) > 0:
3287             command = "branches -u %s" % user
3288         else:
3289             command = "branches"
3290
3291         for info in p4CmdList(command):
3292             details = p4Cmd(["branch", "-o", info["branch"]])
3293             viewIdx = 0
3294             while "View%s" % viewIdx in details:
3295                 paths = details["View%s" % viewIdx].split(" ")
3296                 viewIdx = viewIdx + 1
3297                 # require standard //depot/foo/... //depot/bar/... mapping
3298                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3299                     continue
3300                 source = paths[0]
3301                 destination = paths[1]
3302                 ## HACK
3303                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3304                     source = source[len(self.depotPaths[0]):-4]
3305                     destination = destination[len(self.depotPaths[0]):-4]
3306
3307                     if destination in self.knownBranches:
3308                         if not self.silent:
3309                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3310                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3311                         continue
3312
3313                     self.knownBranches[destination] = source
3314
3315                     lostAndFoundBranches.discard(destination)
3316
3317                     if source not in self.knownBranches:
3318                         lostAndFoundBranches.add(source)
3319
3320         # Perforce does not strictly require branches to be defined, so we also
3321         # check git config for a branch list.
3322         #
3323         # Example of branch definition in git config file:
3324         # [git-p4]
3325         #   branchList=main:branchA
3326         #   branchList=main:branchB
3327         #   branchList=branchA:branchC
3328         configBranches = gitConfigList("git-p4.branchList")
3329         for branch in configBranches:
3330             if branch:
3331                 (source, destination) = branch.split(":")
3332                 self.knownBranches[destination] = source
3333
3334                 lostAndFoundBranches.discard(destination)
3335
3336                 if source not in self.knownBranches:
3337                     lostAndFoundBranches.add(source)
3338
3339
3340         for branch in lostAndFoundBranches:
3341             self.knownBranches[branch] = branch
3342
3343     def getBranchMappingFromGitBranches(self):
3344         branches = p4BranchesInGit(self.importIntoRemotes)
3345         for branch in branches.keys():
3346             if branch == "master":
3347                 branch = "main"
3348             else:
3349                 branch = branch[len(self.projectName):]
3350             self.knownBranches[branch] = branch
3351
3352     def updateOptionDict(self, d):
3353         option_keys = {}
3354         if self.keepRepoPath:
3355             option_keys['keepRepoPath'] = 1
3356
3357         d["options"] = ' '.join(sorted(option_keys.keys()))
3358
3359     def readOptions(self, d):
3360         self.keepRepoPath = ('options' in d
3361                              and ('keepRepoPath' in d['options']))
3362
3363     def gitRefForBranch(self, branch):
3364         if branch == "main":
3365             return self.refPrefix + "master"
3366
3367         if len(branch) <= 0:
3368             return branch
3369
3370         return self.refPrefix + self.projectName + branch
3371
3372     def gitCommitByP4Change(self, ref, change):
3373         if self.verbose:
3374             print("looking in ref " + ref + " for change %s using bisect..." % change)
3375
3376         earliestCommit = ""
3377         latestCommit = parseRevision(ref)
3378
3379         while True:
3380             if self.verbose:
3381                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3382             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3383             if len(next) == 0:
3384                 if self.verbose:
3385                     print("argh")
3386                 return ""
3387             log = extractLogMessageFromGitCommit(next)
3388             settings = extractSettingsGitLog(log)
3389             currentChange = int(settings['change'])
3390             if self.verbose:
3391                 print("current change %s" % currentChange)
3392
3393             if currentChange == change:
3394                 if self.verbose:
3395                     print("found %s" % next)
3396                 return next
3397
3398             if currentChange < change:
3399                 earliestCommit = "^%s" % next
3400             else:
3401                 if next == latestCommit:
3402                     die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3403                 latestCommit = "%s^@" % next
3404
3405         return ""
3406
3407     def importNewBranch(self, branch, maxChange):
3408         # make fast-import flush all changes to disk and update the refs using the checkpoint
3409         # command so that we can try to find the branch parent in the git history
3410         self.gitStream.write("checkpoint\n\n");
3411         self.gitStream.flush();
3412         branchPrefix = self.depotPaths[0] + branch + "/"
3413         range = "@1,%s" % maxChange
3414         #print "prefix" + branchPrefix
3415         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3416         if len(changes) <= 0:
3417             return False
3418         firstChange = changes[0]
3419         #print "first change in branch: %s" % firstChange
3420         sourceBranch = self.knownBranches[branch]
3421         sourceDepotPath = self.depotPaths[0] + sourceBranch
3422         sourceRef = self.gitRefForBranch(sourceBranch)
3423         #print "source " + sourceBranch
3424
3425         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3426         #print "branch parent: %s" % branchParentChange
3427         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3428         if len(gitParent) > 0:
3429             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3430             #print "parent git commit: %s" % gitParent
3431
3432         self.importChanges(changes)
3433         return True
3434
3435     def searchParent(self, parent, branch, target):
3436         parentFound = False
3437         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3438                                      "--no-merges", parent]):
3439             blob = blob.strip()
3440             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3441                 parentFound = True
3442                 if self.verbose:
3443                     print("Found parent of %s in commit %s" % (branch, blob))
3444                 break
3445         if parentFound:
3446             return blob
3447         else:
3448             return None
3449
3450     def importChanges(self, changes, origin_revision=0):
3451         cnt = 1
3452         for change in changes:
3453             description = p4_describe(change)
3454             self.updateOptionDict(description)
3455
3456             if not self.silent:
3457                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3458                 sys.stdout.flush()
3459             cnt = cnt + 1
3460
3461             try:
3462                 if self.detectBranches:
3463                     branches = self.splitFilesIntoBranches(description)
3464                     for branch in branches.keys():
3465                         ## HACK  --hwn
3466                         branchPrefix = self.depotPaths[0] + branch + "/"
3467                         self.branchPrefixes = [ branchPrefix ]
3468
3469                         parent = ""
3470
3471                         filesForCommit = branches[branch]
3472
3473                         if self.verbose:
3474                             print("branch is %s" % branch)
3475
3476                         self.updatedBranches.add(branch)
3477
3478                         if branch not in self.createdBranches:
3479                             self.createdBranches.add(branch)
3480                             parent = self.knownBranches[branch]
3481                             if parent == branch:
3482                                 parent = ""
3483                             else:
3484                                 fullBranch = self.projectName + branch
3485                                 if fullBranch not in self.p4BranchesInGit:
3486                                     if not self.silent:
3487                                         print("\n    Importing new branch %s" % fullBranch);
3488                                     if self.importNewBranch(branch, change - 1):
3489                                         parent = ""
3490                                         self.p4BranchesInGit.append(fullBranch)
3491                                     if not self.silent:
3492                                         print("\n    Resuming with change %s" % change);
3493
3494                                 if self.verbose:
3495                                     print("parent determined through known branches: %s" % parent)
3496
3497                         branch = self.gitRefForBranch(branch)
3498                         parent = self.gitRefForBranch(parent)
3499
3500                         if self.verbose:
3501                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3502
3503                         if len(parent) == 0 and branch in self.initialParents:
3504                             parent = self.initialParents[branch]
3505                             del self.initialParents[branch]
3506
3507                         blob = None
3508                         if len(parent) > 0:
3509                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3510                             if self.verbose:
3511                                 print("Creating temporary branch: " + tempBranch)
3512                             self.commit(description, filesForCommit, tempBranch)
3513                             self.tempBranches.append(tempBranch)
3514                             self.checkpoint()
3515                             blob = self.searchParent(parent, branch, tempBranch)
3516                         if blob:
3517                             self.commit(description, filesForCommit, branch, blob)
3518                         else:
3519                             if self.verbose:
3520                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3521                             self.commit(description, filesForCommit, branch, parent)
3522                 else:
3523                     files = self.extractFilesFromCommit(description)
3524                     self.commit(description, files, self.branch,
3525                                 self.initialParent)
3526                     # only needed once, to connect to the previous commit
3527                     self.initialParent = ""
3528             except IOError:
3529                 print(self.gitError.read())
3530                 sys.exit(1)
3531
3532     def sync_origin_only(self):
3533         if self.syncWithOrigin:
3534             self.hasOrigin = originP4BranchesExist()
3535             if self.hasOrigin:
3536                 if not self.silent:
3537                     print('Syncing with origin first, using "git fetch origin"')
3538                 system("git fetch origin")
3539
3540     def importHeadRevision(self, revision):
3541         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3542
3543         details = {}
3544         details["user"] = "git perforce import user"
3545         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3546                            % (' '.join(self.depotPaths), revision))
3547         details["change"] = revision
3548         newestRevision = 0
3549
3550         fileCnt = 0
3551         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3552
3553         for info in p4CmdList(["files"] + fileArgs):
3554
3555             if 'code' in info and info['code'] == 'error':
3556                 sys.stderr.write("p4 returned an error: %s\n"
3557                                  % info['data'])
3558                 if info['data'].find("must refer to client") >= 0:
3559                     sys.stderr.write("This particular p4 error is misleading.\n")
3560                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3561                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3562                 sys.exit(1)
3563             if 'p4ExitCode' in info:
3564                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3565                 sys.exit(1)
3566
3567
3568             change = int(info["change"])
3569             if change > newestRevision:
3570                 newestRevision = change
3571
3572             if info["action"] in self.delete_actions:
3573                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3574                 #fileCnt = fileCnt + 1
3575                 continue
3576
3577             for prop in ["depotFile", "rev", "action", "type" ]:
3578                 details["%s%s" % (prop, fileCnt)] = info[prop]
3579
3580             fileCnt = fileCnt + 1
3581
3582         details["change"] = newestRevision
3583
3584         # Use time from top-most change so that all git p4 clones of
3585         # the same p4 repo have the same commit SHA1s.
3586         res = p4_describe(newestRevision)
3587         details["time"] = res["time"]
3588
3589         self.updateOptionDict(details)
3590         try:
3591             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3592         except IOError as err:
3593             print("IO error with git fast-import. Is your git version recent enough?")
3594             print("IO error details: {}".format(err))
3595             print(self.gitError.read())
3596
3597
3598     def importRevisions(self, args, branch_arg_given):
3599         changes = []
3600
3601         if len(self.changesFile) > 0:
3602             with open(self.changesFile) as f:
3603                 output = f.readlines()
3604             changeSet = set()
3605             for line in output:
3606                 changeSet.add(int(line))
3607
3608             for change in changeSet:
3609                 changes.append(change)
3610
3611             changes.sort()
3612         else:
3613             # catch "git p4 sync" with no new branches, in a repo that
3614             # does not have any existing p4 branches
3615             if len(args) == 0:
3616                 if not self.p4BranchesInGit:
3617                     raise P4CommandException("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3618
3619                 # The default branch is master, unless --branch is used to
3620                 # specify something else.  Make sure it exists, or complain
3621                 # nicely about how to use --branch.
3622                 if not self.detectBranches:
3623                     if not branch_exists(self.branch):
3624                         if branch_arg_given:
3625                             raise P4CommandException("Error: branch %s does not exist." % self.branch)
3626                         else:
3627                             raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3628                                 self.branch)
3629
3630             if self.verbose:
3631                 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3632                                                           self.changeRange))
3633             changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3634
3635             if len(self.maxChanges) > 0:
3636                 changes = changes[:min(int(self.maxChanges), len(changes))]
3637
3638         if len(changes) == 0:
3639             if not self.silent:
3640                 print("No changes to import!")
3641         else:
3642             if not self.silent and not self.detectBranches:
3643                 print("Import destination: %s" % self.branch)
3644
3645             self.updatedBranches = set()
3646
3647             if not self.detectBranches:
3648                 if args:
3649                     # start a new branch
3650                     self.initialParent = ""
3651                 else:
3652                     # build on a previous revision
3653                     self.initialParent = parseRevision(self.branch)
3654
3655             self.importChanges(changes)
3656
3657             if not self.silent:
3658                 print("")
3659                 if len(self.updatedBranches) > 0:
3660                     sys.stdout.write("Updated branches: ")
3661                     for b in self.updatedBranches:
3662                         sys.stdout.write("%s " % b)
3663                     sys.stdout.write("\n")
3664
3665     def openStreams(self):
3666         self.importProcess = subprocess.Popen(["git", "fast-import"],
3667                                               stdin=subprocess.PIPE,
3668                                               stdout=subprocess.PIPE,
3669                                               stderr=subprocess.PIPE);
3670         self.gitOutput = self.importProcess.stdout
3671         self.gitStream = self.importProcess.stdin
3672         self.gitError = self.importProcess.stderr
3673
3674         if bytes is not str:
3675             # Wrap gitStream.write() so that it can be called using `str` arguments
3676             def make_encoded_write(write):
3677                 def encoded_write(s):
3678                     return write(s.encode() if isinstance(s, str) else s)
3679                 return encoded_write
3680
3681             self.gitStream.write = make_encoded_write(self.gitStream.write)
3682
3683     def closeStreams(self):
3684         if self.gitStream is None:
3685             return
3686         self.gitStream.close()
3687         if self.importProcess.wait() != 0:
3688             die("fast-import failed: %s" % self.gitError.read())
3689         self.gitOutput.close()
3690         self.gitError.close()
3691         self.gitStream = None
3692
3693     def run(self, args):
3694         if self.importIntoRemotes:
3695             self.refPrefix = "refs/remotes/p4/"
3696         else:
3697             self.refPrefix = "refs/heads/p4/"
3698
3699         self.sync_origin_only()
3700
3701         branch_arg_given = bool(self.branch)
3702         if len(self.branch) == 0:
3703             self.branch = self.refPrefix + "master"
3704             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3705                 system("git update-ref %s refs/heads/p4" % self.branch)
3706                 system("git branch -D p4")
3707
3708         # accept either the command-line option, or the configuration variable
3709         if self.useClientSpec:
3710             # will use this after clone to set the variable
3711             self.useClientSpec_from_options = True
3712         else:
3713             if gitConfigBool("git-p4.useclientspec"):
3714                 self.useClientSpec = True
3715         if self.useClientSpec:
3716             self.clientSpecDirs = getClientSpec()
3717
3718         # TODO: should always look at previous commits,
3719         # merge with previous imports, if possible.
3720         if args == []:
3721             if self.hasOrigin:
3722                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3723
3724             # branches holds mapping from branch name to sha1
3725             branches = p4BranchesInGit(self.importIntoRemotes)
3726
3727             # restrict to just this one, disabling detect-branches
3728             if branch_arg_given:
3729                 short = self.branch.split("/")[-1]
3730                 if short in branches:
3731                     self.p4BranchesInGit = [ short ]
3732             else:
3733                 self.p4BranchesInGit = branches.keys()
3734
3735             if len(self.p4BranchesInGit) > 1:
3736                 if not self.silent:
3737                     print("Importing from/into multiple branches")
3738                 self.detectBranches = True
3739                 for branch in branches.keys():
3740                     self.initialParents[self.refPrefix + branch] = \
3741                         branches[branch]
3742
3743             if self.verbose:
3744                 print("branches: %s" % self.p4BranchesInGit)
3745
3746             p4Change = 0
3747             for branch in self.p4BranchesInGit:
3748                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3749
3750                 settings = extractSettingsGitLog(logMsg)
3751
3752                 self.readOptions(settings)
3753                 if ('depot-paths' in settings
3754                     and 'change' in settings):
3755                     change = int(settings['change']) + 1
3756                     p4Change = max(p4Change, change)
3757
3758                     depotPaths = sorted(settings['depot-paths'])
3759                     if self.previousDepotPaths == []:
3760                         self.previousDepotPaths = depotPaths
3761                     else:
3762                         paths = []
3763                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3764                             prev_list = prev.split("/")
3765                             cur_list = cur.split("/")
3766                             for i in range(0, min(len(cur_list), len(prev_list))):
3767                                 if cur_list[i] != prev_list[i]:
3768                                     i = i - 1
3769                                     break
3770
3771                             paths.append ("/".join(cur_list[:i + 1]))
3772
3773                         self.previousDepotPaths = paths
3774
3775             if p4Change > 0:
3776                 self.depotPaths = sorted(self.previousDepotPaths)
3777                 self.changeRange = "@%s,#head" % p4Change
3778                 if not self.silent and not self.detectBranches:
3779                     print("Performing incremental import into %s git branch" % self.branch)
3780
3781         # accept multiple ref name abbreviations:
3782         #    refs/foo/bar/branch -> use it exactly
3783         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3784         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3785         if not self.branch.startswith("refs/"):
3786             if self.importIntoRemotes:
3787                 prepend = "refs/remotes/"
3788             else:
3789                 prepend = "refs/heads/"
3790             if not self.branch.startswith("p4/"):
3791                 prepend += "p4/"
3792             self.branch = prepend + self.branch
3793
3794         if len(args) == 0 and self.depotPaths:
3795             if not self.silent:
3796                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3797         else:
3798             if self.depotPaths and self.depotPaths != args:
3799                 print("previous import used depot path %s and now %s was specified. "
3800                        "This doesn't work!" % (' '.join (self.depotPaths),
3801                                                ' '.join (args)))
3802                 sys.exit(1)
3803
3804             self.depotPaths = sorted(args)
3805
3806         revision = ""
3807         self.users = {}
3808
3809         # Make sure no revision specifiers are used when --changesfile
3810         # is specified.
3811         bad_changesfile = False
3812         if len(self.changesFile) > 0:
3813             for p in self.depotPaths:
3814                 if p.find("@") >= 0 or p.find("#") >= 0:
3815                     bad_changesfile = True
3816                     break
3817         if bad_changesfile:
3818             die("Option --changesfile is incompatible with revision specifiers")
3819
3820         newPaths = []
3821         for p in self.depotPaths:
3822             if p.find("@") != -1:
3823                 atIdx = p.index("@")
3824                 self.changeRange = p[atIdx:]
3825                 if self.changeRange == "@all":
3826                     self.changeRange = ""
3827                 elif ',' not in self.changeRange:
3828                     revision = self.changeRange
3829                     self.changeRange = ""
3830                 p = p[:atIdx]
3831             elif p.find("#") != -1:
3832                 hashIdx = p.index("#")
3833                 revision = p[hashIdx:]
3834                 p = p[:hashIdx]
3835             elif self.previousDepotPaths == []:
3836                 # pay attention to changesfile, if given, else import
3837                 # the entire p4 tree at the head revision
3838                 if len(self.changesFile) == 0:
3839                     revision = "#head"
3840
3841             p = re.sub ("\.\.\.$", "", p)
3842             if not p.endswith("/"):
3843                 p += "/"
3844
3845             newPaths.append(p)
3846
3847         self.depotPaths = newPaths
3848
3849         # --detect-branches may change this for each branch
3850         self.branchPrefixes = self.depotPaths
3851
3852         self.loadUserMapFromCache()
3853         self.labels = {}
3854         if self.detectLabels:
3855             self.getLabels();
3856
3857         if self.detectBranches:
3858             ## FIXME - what's a P4 projectName ?
3859             self.projectName = self.guessProjectName()
3860
3861             if self.hasOrigin:
3862                 self.getBranchMappingFromGitBranches()
3863             else:
3864                 self.getBranchMapping()
3865             if self.verbose:
3866                 print("p4-git branches: %s" % self.p4BranchesInGit)
3867                 print("initial parents: %s" % self.initialParents)
3868             for b in self.p4BranchesInGit:
3869                 if b != "master":
3870
3871                     ## FIXME
3872                     b = b[len(self.projectName):]
3873                 self.createdBranches.add(b)
3874
3875         p4_check_access()
3876
3877         self.openStreams()
3878
3879         err = None
3880
3881         try:
3882             if revision:
3883                 self.importHeadRevision(revision)
3884             else:
3885                 self.importRevisions(args, branch_arg_given)
3886
3887             if gitConfigBool("git-p4.importLabels"):
3888                 self.importLabels = True
3889
3890             if self.importLabels:
3891                 p4Labels = getP4Labels(self.depotPaths)
3892                 gitTags = getGitTags()
3893
3894                 missingP4Labels = p4Labels - gitTags
3895                 self.importP4Labels(self.gitStream, missingP4Labels)
3896
3897         except P4CommandException as e:
3898             err = e
3899
3900         finally:
3901             self.closeStreams()
3902
3903         if err:
3904             die(str(err))
3905
3906         # Cleanup temporary branches created during import
3907         if self.tempBranches != []:
3908             for branch in self.tempBranches:
3909                 read_pipe("git update-ref -d %s" % branch)
3910             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3911
3912         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3913         # a convenient shortcut refname "p4".
3914         if self.importIntoRemotes:
3915             head_ref = self.refPrefix + "HEAD"
3916             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3917                 system(["git", "symbolic-ref", head_ref, self.branch])
3918
3919         return True
3920
3921 class P4Rebase(Command):
3922     def __init__(self):
3923         Command.__init__(self)
3924         self.options = [
3925                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3926         ]
3927         self.importLabels = False
3928         self.description = ("Fetches the latest revision from perforce and "
3929                             + "rebases the current work (branch) against it")
3930
3931     def run(self, args):
3932         sync = P4Sync()
3933         sync.importLabels = self.importLabels
3934         sync.run([])
3935
3936         return self.rebase()
3937
3938     def rebase(self):
3939         if os.system("git update-index --refresh") != 0:
3940             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3941         if len(read_pipe("git diff-index HEAD --")) > 0:
3942             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3943
3944         [upstream, settings] = findUpstreamBranchPoint()
3945         if len(upstream) == 0:
3946             die("Cannot find upstream branchpoint for rebase")
3947
3948         # the branchpoint may be p4/foo~3, so strip off the parent
3949         upstream = re.sub("~[0-9]+$", "", upstream)
3950
3951         print("Rebasing the current branch onto %s" % upstream)
3952         oldHead = read_pipe("git rev-parse HEAD").strip()
3953         system("git rebase %s" % upstream)
3954         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3955         return True
3956
3957 class P4Clone(P4Sync):
3958     def __init__(self):
3959         P4Sync.__init__(self)
3960         self.description = "Creates a new git repository and imports from Perforce into it"
3961         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3962         self.options += [
3963             optparse.make_option("--destination", dest="cloneDestination",
3964                                  action='store', default=None,
3965                                  help="where to leave result of the clone"),
3966             optparse.make_option("--bare", dest="cloneBare",
3967                                  action="store_true", default=False),
3968         ]
3969         self.cloneDestination = None
3970         self.needsGit = False
3971         self.cloneBare = False
3972
3973     def defaultDestination(self, args):
3974         ## TODO: use common prefix of args?
3975         depotPath = args[0]
3976         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3977         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3978         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3979         depotDir = re.sub(r"/$", "", depotDir)
3980         return os.path.split(depotDir)[1]
3981
3982     def run(self, args):
3983         if len(args) < 1:
3984             return False
3985
3986         if self.keepRepoPath and not self.cloneDestination:
3987             sys.stderr.write("Must specify destination for --keep-path\n")
3988             sys.exit(1)
3989
3990         depotPaths = args
3991
3992         if not self.cloneDestination and len(depotPaths) > 1:
3993             self.cloneDestination = depotPaths[-1]
3994             depotPaths = depotPaths[:-1]
3995
3996         for p in depotPaths:
3997             if not p.startswith("//"):
3998                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3999                 return False
4000
4001         if not self.cloneDestination:
4002             self.cloneDestination = self.defaultDestination(args)
4003
4004         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
4005
4006         if not os.path.exists(self.cloneDestination):
4007             os.makedirs(self.cloneDestination)
4008         chdir(self.cloneDestination)
4009
4010         init_cmd = [ "git", "init" ]
4011         if self.cloneBare:
4012             init_cmd.append("--bare")
4013         retcode = subprocess.call(init_cmd)
4014         if retcode:
4015             raise CalledProcessError(retcode, init_cmd)
4016
4017         if not P4Sync.run(self, depotPaths):
4018             return False
4019
4020         # create a master branch and check out a work tree
4021         if gitBranchExists(self.branch):
4022             system([ "git", "branch", "master", self.branch ])
4023             if not self.cloneBare:
4024                 system([ "git", "checkout", "-f" ])
4025         else:
4026             print('Not checking out any branch, use ' \
4027                   '"git checkout -q -b master <branch>"')
4028
4029         # auto-set this variable if invoked with --use-client-spec
4030         if self.useClientSpec_from_options:
4031             system("git config --bool git-p4.useclientspec true")
4032
4033         return True
4034
4035 class P4Unshelve(Command):
4036     def __init__(self):
4037         Command.__init__(self)
4038         self.options = []
4039         self.origin = "HEAD"
4040         self.description = "Unshelve a P4 changelist into a git commit"
4041         self.usage = "usage: %prog [options] changelist"
4042         self.options += [
4043                 optparse.make_option("--origin", dest="origin",
4044                     help="Use this base revision instead of the default (%s)" % self.origin),
4045         ]
4046         self.verbose = False
4047         self.noCommit = False
4048         self.destbranch = "refs/remotes/p4-unshelved"
4049
4050     def renameBranch(self, branch_name):
4051         """ Rename the existing branch to branch_name.N
4052         """
4053
4054         found = True
4055         for i in range(0,1000):
4056             backup_branch_name = "{0}.{1}".format(branch_name, i)
4057             if not gitBranchExists(backup_branch_name):
4058                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
4059                 gitDeleteRef(branch_name)
4060                 found = True
4061                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
4062                 break
4063
4064         if not found:
4065             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
4066
4067     def findLastP4Revision(self, starting_point):
4068         """ Look back from starting_point for the first commit created by git-p4
4069             to find the P4 commit we are based on, and the depot-paths.
4070         """
4071
4072         for parent in (range(65535)):
4073             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
4074             settings = extractSettingsGitLog(log)
4075             if 'change' in settings:
4076                 return settings
4077
4078         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4079
4080     def createShelveParent(self, change, branch_name, sync, origin):
4081         """ Create a commit matching the parent of the shelved changelist 'change'
4082         """
4083         parent_description = p4_describe(change, shelved=True)
4084         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4085         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4086
4087         parent_files = []
4088         for f in files:
4089             # if it was added in the shelved changelist, it won't exist in the parent
4090             if f['action'] in self.add_actions:
4091                 continue
4092
4093             # if it was deleted in the shelved changelist it must not be deleted
4094             # in the parent - we might even need to create it if the origin branch
4095             # does not have it
4096             if f['action'] in self.delete_actions:
4097                 f['action'] = 'add'
4098
4099             parent_files.append(f)
4100
4101         sync.commit(parent_description, parent_files, branch_name,
4102                 parent=origin, allow_empty=True)
4103         print("created parent commit for {0} based on {1} in {2}".format(
4104             change, self.origin, branch_name))
4105
4106     def run(self, args):
4107         if len(args) != 1:
4108             return False
4109
4110         if not gitBranchExists(self.origin):
4111             sys.exit("origin branch {0} does not exist".format(self.origin))
4112
4113         sync = P4Sync()
4114         changes = args
4115
4116         # only one change at a time
4117         change = changes[0]
4118
4119         # if the target branch already exists, rename it
4120         branch_name = "{0}/{1}".format(self.destbranch, change)
4121         if gitBranchExists(branch_name):
4122             self.renameBranch(branch_name)
4123         sync.branch = branch_name
4124
4125         sync.verbose = self.verbose
4126         sync.suppress_meta_comment = True
4127
4128         settings = self.findLastP4Revision(self.origin)
4129         sync.depotPaths = settings['depot-paths']
4130         sync.branchPrefixes = sync.depotPaths
4131
4132         sync.openStreams()
4133         sync.loadUserMapFromCache()
4134         sync.silent = True
4135
4136         # create a commit for the parent of the shelved changelist
4137         self.createShelveParent(change, branch_name, sync, self.origin)
4138
4139         # create the commit for the shelved changelist itself
4140         description = p4_describe(change, True)
4141         files = sync.extractFilesFromCommit(description, True, change)
4142
4143         sync.commit(description, files, branch_name, "")
4144         sync.closeStreams()
4145
4146         print("unshelved changelist {0} into {1}".format(change, branch_name))
4147
4148         return True
4149
4150 class P4Branches(Command):
4151     def __init__(self):
4152         Command.__init__(self)
4153         self.options = [ ]
4154         self.description = ("Shows the git branches that hold imports and their "
4155                             + "corresponding perforce depot paths")
4156         self.verbose = False
4157
4158     def run(self, args):
4159         if originP4BranchesExist():
4160             createOrUpdateBranchesFromOrigin()
4161
4162         cmdline = "git rev-parse --symbolic "
4163         cmdline += " --remotes"
4164
4165         for line in read_pipe_lines(cmdline):
4166             line = line.strip()
4167
4168             if not line.startswith('p4/') or line == "p4/HEAD":
4169                 continue
4170             branch = line
4171
4172             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4173             settings = extractSettingsGitLog(log)
4174
4175             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4176         return True
4177
4178 class HelpFormatter(optparse.IndentedHelpFormatter):
4179     def __init__(self):
4180         optparse.IndentedHelpFormatter.__init__(self)
4181
4182     def format_description(self, description):
4183         if description:
4184             return description + "\n"
4185         else:
4186             return ""
4187
4188 def printUsage(commands):
4189     print("usage: %s <command> [options]" % sys.argv[0])
4190     print("")
4191     print("valid commands: %s" % ", ".join(commands))
4192     print("")
4193     print("Try %s <command> --help for command specific help." % sys.argv[0])
4194     print("")
4195
4196 commands = {
4197     "debug" : P4Debug,
4198     "submit" : P4Submit,
4199     "commit" : P4Submit,
4200     "sync" : P4Sync,
4201     "rebase" : P4Rebase,
4202     "clone" : P4Clone,
4203     "rollback" : P4RollBack,
4204     "branches" : P4Branches,
4205     "unshelve" : P4Unshelve,
4206 }
4207
4208
4209 def main():
4210     if len(sys.argv[1:]) == 0:
4211         printUsage(commands.keys())
4212         sys.exit(2)
4213
4214     cmdName = sys.argv[1]
4215     try:
4216         klass = commands[cmdName]
4217         cmd = klass()
4218     except KeyError:
4219         print("unknown command %s" % cmdName)
4220         print("")
4221         printUsage(commands.keys())
4222         sys.exit(2)
4223
4224     options = cmd.options
4225     cmd.gitdir = os.environ.get("GIT_DIR", None)
4226
4227     args = sys.argv[2:]
4228
4229     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4230     if cmd.needsGit:
4231         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4232
4233     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4234                                    options,
4235                                    description = cmd.description,
4236                                    formatter = HelpFormatter())
4237
4238     try:
4239         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4240     except:
4241         parser.print_help()
4242         raise
4243
4244     global verbose
4245     verbose = cmd.verbose
4246     if cmd.needsGit:
4247         if cmd.gitdir == None:
4248             cmd.gitdir = os.path.abspath(".git")
4249             if not isValidGitDir(cmd.gitdir):
4250                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4251                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4252                 if os.path.exists(cmd.gitdir):
4253                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4254                     if len(cdup) > 0:
4255                         chdir(cdup);
4256
4257         if not isValidGitDir(cmd.gitdir):
4258             if isValidGitDir(cmd.gitdir + "/.git"):
4259                 cmd.gitdir += "/.git"
4260             else:
4261                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4262
4263         # so git commands invoked from the P4 workspace will succeed
4264         os.environ["GIT_DIR"] = cmd.gitdir
4265
4266     if not cmd.run(args):
4267         parser.print_help()
4268         sys.exit(2)
4269
4270
4271 if __name__ == '__main__':
4272     main()