word diff: handle zero length matches
[git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
17 #
18 import sys
19 if sys.version_info.major < 3 and sys.version_info.minor < 7:
20     sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
21     sys.exit(1)
22 import os
23 import optparse
24 import functools
25 import marshal
26 import subprocess
27 import tempfile
28 import time
29 import platform
30 import re
31 import shutil
32 import stat
33 import zipfile
34 import zlib
35 import ctypes
36 import errno
37 import glob
38
39 # On python2.7 where raw_input() and input() are both availble,
40 # we want raw_input's semantics, but aliased to input for python3
41 # compatibility
42 # support basestring in python3
43 try:
44     if raw_input and input:
45         input = raw_input
46 except:
47     pass
48
49 verbose = False
50
51 # Only labels/tags matching this will be imported/exported
52 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
53
54 # The block size is reduced automatically if required
55 defaultBlockSize = 1<<20
56
57 p4_access_checked = False
58
59 def p4_build_cmd(cmd):
60     """Build a suitable p4 command line.
61
62     This consolidates building and returning a p4 command line into one
63     location. It means that hooking into the environment, or other configuration
64     can be done more easily.
65     """
66     real_cmd = ["p4"]
67
68     user = gitConfig("git-p4.user")
69     if len(user) > 0:
70         real_cmd += ["-u",user]
71
72     password = gitConfig("git-p4.password")
73     if len(password) > 0:
74         real_cmd += ["-P", password]
75
76     port = gitConfig("git-p4.port")
77     if len(port) > 0:
78         real_cmd += ["-p", port]
79
80     host = gitConfig("git-p4.host")
81     if len(host) > 0:
82         real_cmd += ["-H", host]
83
84     client = gitConfig("git-p4.client")
85     if len(client) > 0:
86         real_cmd += ["-c", client]
87
88     retries = gitConfigInt("git-p4.retries")
89     if retries is None:
90         # Perform 3 retries by default
91         retries = 3
92     if retries > 0:
93         # Provide a way to not pass this option by setting git-p4.retries to 0
94         real_cmd += ["-r", str(retries)]
95
96     if not isinstance(cmd, list):
97         real_cmd = ' '.join(real_cmd) + ' ' + cmd
98     else:
99         real_cmd += cmd
100
101     # now check that we can actually talk to the server
102     global p4_access_checked
103     if not p4_access_checked:
104         p4_access_checked = True    # suppress access checks in p4_check_access itself
105         p4_check_access()
106
107     return real_cmd
108
109 def git_dir(path):
110     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
111         This won't automatically add ".git" to a directory.
112     """
113     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
114     if not d or len(d) == 0:
115         return None
116     else:
117         return d
118
119 def chdir(path, is_client_path=False):
120     """Do chdir to the given path, and set the PWD environment
121        variable for use by P4.  It does not look at getcwd() output.
122        Since we're not using the shell, it is necessary to set the
123        PWD environment variable explicitly.
124
125        Normally, expand the path to force it to be absolute.  This
126        addresses the use of relative path names inside P4 settings,
127        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
128        as given; it looks for .p4config using PWD.
129
130        If is_client_path, the path was handed to us directly by p4,
131        and may be a symbolic link.  Do not call os.getcwd() in this
132        case, because it will cause p4 to think that PWD is not inside
133        the client path.
134        """
135
136     os.chdir(path)
137     if not is_client_path:
138         path = os.getcwd()
139     os.environ['PWD'] = path
140
141 def calcDiskFree():
142     """Return free space in bytes on the disk of the given dirname."""
143     if platform.system() == 'Windows':
144         free_bytes = ctypes.c_ulonglong(0)
145         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
146         return free_bytes.value
147     else:
148         st = os.statvfs(os.getcwd())
149         return st.f_bavail * st.f_frsize
150
151 def die(msg):
152     """ Terminate execution. Make sure that any running child processes have been wait()ed for before
153         calling this.
154     """
155     if verbose:
156         raise Exception(msg)
157     else:
158         sys.stderr.write(msg + "\n")
159         sys.exit(1)
160
161 def prompt(prompt_text):
162     """ Prompt the user to choose one of the choices
163
164     Choices are identified in the prompt_text by square brackets around
165     a single letter option.
166     """
167     choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
168     while True:
169         sys.stderr.flush()
170         sys.stdout.write(prompt_text)
171         sys.stdout.flush()
172         response=sys.stdin.readline().strip().lower()
173         if not response:
174             continue
175         response = response[0]
176         if response in choices:
177             return response
178
179 # We need different encoding/decoding strategies for text data being passed
180 # around in pipes depending on python version
181 if bytes is not str:
182     # For python3, always encode and decode as appropriate
183     def decode_text_stream(s):
184         return s.decode() if isinstance(s, bytes) else s
185     def encode_text_stream(s):
186         return s.encode() if isinstance(s, str) else s
187 else:
188     # For python2.7, pass read strings as-is, but also allow writing unicode
189     def decode_text_stream(s):
190         return s
191     def encode_text_stream(s):
192         return s.encode('utf_8') if isinstance(s, unicode) else s
193
194 def decode_path(path):
195     """Decode a given string (bytes or otherwise) using configured path encoding options
196     """
197     encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
198     if bytes is not str:
199         return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
200     else:
201         try:
202             path.decode('ascii')
203         except:
204             path = path.decode(encoding, errors='replace')
205             if verbose:
206                 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
207         return path
208
209 def run_git_hook(cmd, param=[]):
210     """Execute a hook if the hook exists."""
211     if verbose:
212         sys.stderr.write("Looking for hook: %s\n" % cmd)
213         sys.stderr.flush()
214
215     hooks_path = gitConfig("core.hooksPath")
216     if len(hooks_path) <= 0:
217         hooks_path = os.path.join(os.environ["GIT_DIR"], "hooks")
218
219     if not isinstance(param, list):
220         param=[param]
221
222     # resolve hook file name, OS depdenent
223     hook_file = os.path.join(hooks_path, cmd)
224     if platform.system() == 'Windows':
225         if not os.path.isfile(hook_file):
226             # look for the file with an extension
227             files = glob.glob(hook_file + ".*")
228             if not files:
229                 return True
230             files.sort()
231             hook_file = files.pop()
232             while hook_file.upper().endswith(".SAMPLE"):
233                 # The file is a sample hook. We don't want it
234                 if len(files) > 0:
235                     hook_file = files.pop()
236                 else:
237                     return True
238
239     if not os.path.isfile(hook_file) or not os.access(hook_file, os.X_OK):
240         return True
241
242     return run_hook_command(hook_file, param) == 0
243
244 def run_hook_command(cmd, param):
245     """Executes a git hook command
246        cmd = the command line file to be executed. This can be
247        a file that is run by OS association.
248
249        param = a list of parameters to pass to the cmd command
250
251        On windows, the extension is checked to see if it should
252        be run with the Git for Windows Bash shell.  If there
253        is no file extension, the file is deemed a bash shell
254        and will be handed off to sh.exe. Otherwise, Windows
255        will be called with the shell to handle the file assocation.
256
257        For non Windows operating systems, the file is called
258        as an executable.
259     """
260     cli = [cmd] + param
261     use_shell = False
262     if platform.system() == 'Windows':
263         (root,ext) = os.path.splitext(cmd)
264         if ext == "":
265             exe_path = os.environ.get("EXEPATH")
266             if exe_path is None:
267                 exe_path = ""
268             else:
269                 exe_path = os.path.join(exe_path, "bin")
270             cli = [os.path.join(exe_path, "SH.EXE")] + cli
271         else:
272             use_shell = True
273     return subprocess.call(cli, shell=use_shell)
274
275
276 def write_pipe(c, stdin):
277     if verbose:
278         sys.stderr.write('Writing pipe: %s\n' % str(c))
279
280     expand = not isinstance(c, list)
281     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
282     pipe = p.stdin
283     val = pipe.write(stdin)
284     pipe.close()
285     if p.wait():
286         die('Command failed: %s' % str(c))
287
288     return val
289
290 def p4_write_pipe(c, stdin):
291     real_cmd = p4_build_cmd(c)
292     if bytes is not str and isinstance(stdin, str):
293         stdin = encode_text_stream(stdin)
294     return write_pipe(real_cmd, stdin)
295
296 def read_pipe_full(c):
297     """ Read output from  command. Returns a tuple
298         of the return status, stdout text and stderr
299         text.
300     """
301     if verbose:
302         sys.stderr.write('Reading pipe: %s\n' % str(c))
303
304     expand = not isinstance(c, list)
305     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
306     (out, err) = p.communicate()
307     return (p.returncode, out, decode_text_stream(err))
308
309 def read_pipe(c, ignore_error=False, raw=False):
310     """ Read output from  command. Returns the output text on
311         success. On failure, terminates execution, unless
312         ignore_error is True, when it returns an empty string.
313
314         If raw is True, do not attempt to decode output text.
315     """
316     (retcode, out, err) = read_pipe_full(c)
317     if retcode != 0:
318         if ignore_error:
319             out = ""
320         else:
321             die('Command failed: %s\nError: %s' % (str(c), err))
322     if not raw:
323         out = decode_text_stream(out)
324     return out
325
326 def read_pipe_text(c):
327     """ Read output from a command with trailing whitespace stripped.
328         On error, returns None.
329     """
330     (retcode, out, err) = read_pipe_full(c)
331     if retcode != 0:
332         return None
333     else:
334         return decode_text_stream(out).rstrip()
335
336 def p4_read_pipe(c, ignore_error=False, raw=False):
337     real_cmd = p4_build_cmd(c)
338     return read_pipe(real_cmd, ignore_error, raw=raw)
339
340 def read_pipe_lines(c):
341     if verbose:
342         sys.stderr.write('Reading pipe: %s\n' % str(c))
343
344     expand = not isinstance(c, list)
345     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
346     pipe = p.stdout
347     val = [decode_text_stream(line) for line in pipe.readlines()]
348     if pipe.close() or p.wait():
349         die('Command failed: %s' % str(c))
350     return val
351
352 def p4_read_pipe_lines(c):
353     """Specifically invoke p4 on the command supplied. """
354     real_cmd = p4_build_cmd(c)
355     return read_pipe_lines(real_cmd)
356
357 def p4_has_command(cmd):
358     """Ask p4 for help on this command.  If it returns an error, the
359        command does not exist in this version of p4."""
360     real_cmd = p4_build_cmd(["help", cmd])
361     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
362                                    stderr=subprocess.PIPE)
363     p.communicate()
364     return p.returncode == 0
365
366 def p4_has_move_command():
367     """See if the move command exists, that it supports -k, and that
368        it has not been administratively disabled.  The arguments
369        must be correct, but the filenames do not have to exist.  Use
370        ones with wildcards so even if they exist, it will fail."""
371
372     if not p4_has_command("move"):
373         return False
374     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
375     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
376     (out, err) = p.communicate()
377     err = decode_text_stream(err)
378     # return code will be 1 in either case
379     if err.find("Invalid option") >= 0:
380         return False
381     if err.find("disabled") >= 0:
382         return False
383     # assume it failed because @... was invalid changelist
384     return True
385
386 def system(cmd, ignore_error=False):
387     expand = not isinstance(cmd, list)
388     if verbose:
389         sys.stderr.write("executing %s\n" % str(cmd))
390     retcode = subprocess.call(cmd, shell=expand)
391     if retcode and not ignore_error:
392         raise CalledProcessError(retcode, cmd)
393
394     return retcode
395
396 def p4_system(cmd):
397     """Specifically invoke p4 as the system command. """
398     real_cmd = p4_build_cmd(cmd)
399     expand = not isinstance(real_cmd, list)
400     retcode = subprocess.call(real_cmd, shell=expand)
401     if retcode:
402         raise CalledProcessError(retcode, real_cmd)
403
404 def die_bad_access(s):
405     die("failure accessing depot: {0}".format(s.rstrip()))
406
407 def p4_check_access(min_expiration=1):
408     """ Check if we can access Perforce - account still logged in
409     """
410     results = p4CmdList(["login", "-s"])
411
412     if len(results) == 0:
413         # should never get here: always get either some results, or a p4ExitCode
414         assert("could not parse response from perforce")
415
416     result = results[0]
417
418     if 'p4ExitCode' in result:
419         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
420         die_bad_access("could not run p4")
421
422     code = result.get("code")
423     if not code:
424         # we get here if we couldn't connect and there was nothing to unmarshal
425         die_bad_access("could not connect")
426
427     elif code == "stat":
428         expiry = result.get("TicketExpiration")
429         if expiry:
430             expiry = int(expiry)
431             if expiry > min_expiration:
432                 # ok to carry on
433                 return
434             else:
435                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
436
437         else:
438             # account without a timeout - all ok
439             return
440
441     elif code == "error":
442         data = result.get("data")
443         if data:
444             die_bad_access("p4 error: {0}".format(data))
445         else:
446             die_bad_access("unknown error")
447     elif code == "info":
448         return
449     else:
450         die_bad_access("unknown error code {0}".format(code))
451
452 _p4_version_string = None
453 def p4_version_string():
454     """Read the version string, showing just the last line, which
455        hopefully is the interesting version bit.
456
457        $ p4 -V
458        Perforce - The Fast Software Configuration Management System.
459        Copyright 1995-2011 Perforce Software.  All rights reserved.
460        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
461     """
462     global _p4_version_string
463     if not _p4_version_string:
464         a = p4_read_pipe_lines(["-V"])
465         _p4_version_string = a[-1].rstrip()
466     return _p4_version_string
467
468 def p4_integrate(src, dest):
469     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
470
471 def p4_sync(f, *options):
472     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
473
474 def p4_add(f):
475     # forcibly add file names with wildcards
476     if wildcard_present(f):
477         p4_system(["add", "-f", f])
478     else:
479         p4_system(["add", f])
480
481 def p4_delete(f):
482     p4_system(["delete", wildcard_encode(f)])
483
484 def p4_edit(f, *options):
485     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
486
487 def p4_revert(f):
488     p4_system(["revert", wildcard_encode(f)])
489
490 def p4_reopen(type, f):
491     p4_system(["reopen", "-t", type, wildcard_encode(f)])
492
493 def p4_reopen_in_change(changelist, files):
494     cmd = ["reopen", "-c", str(changelist)] + files
495     p4_system(cmd)
496
497 def p4_move(src, dest):
498     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
499
500 def p4_last_change():
501     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
502     return int(results[0]['change'])
503
504 def p4_describe(change, shelved=False):
505     """Make sure it returns a valid result by checking for
506        the presence of field "time".  Return a dict of the
507        results."""
508
509     cmd = ["describe", "-s"]
510     if shelved:
511         cmd += ["-S"]
512     cmd += [str(change)]
513
514     ds = p4CmdList(cmd, skip_info=True)
515     if len(ds) != 1:
516         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
517
518     d = ds[0]
519
520     if "p4ExitCode" in d:
521         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
522                                                       str(d)))
523     if "code" in d:
524         if d["code"] == "error":
525             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
526
527     if "time" not in d:
528         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
529
530     return d
531
532 #
533 # Canonicalize the p4 type and return a tuple of the
534 # base type, plus any modifiers.  See "p4 help filetypes"
535 # for a list and explanation.
536 #
537 def split_p4_type(p4type):
538
539     p4_filetypes_historical = {
540         "ctempobj": "binary+Sw",
541         "ctext": "text+C",
542         "cxtext": "text+Cx",
543         "ktext": "text+k",
544         "kxtext": "text+kx",
545         "ltext": "text+F",
546         "tempobj": "binary+FSw",
547         "ubinary": "binary+F",
548         "uresource": "resource+F",
549         "uxbinary": "binary+Fx",
550         "xbinary": "binary+x",
551         "xltext": "text+Fx",
552         "xtempobj": "binary+Swx",
553         "xtext": "text+x",
554         "xunicode": "unicode+x",
555         "xutf16": "utf16+x",
556     }
557     if p4type in p4_filetypes_historical:
558         p4type = p4_filetypes_historical[p4type]
559     mods = ""
560     s = p4type.split("+")
561     base = s[0]
562     mods = ""
563     if len(s) > 1:
564         mods = s[1]
565     return (base, mods)
566
567 #
568 # return the raw p4 type of a file (text, text+ko, etc)
569 #
570 def p4_type(f):
571     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
572     return results[0]['headType']
573
574 #
575 # Given a type base and modifier, return a regexp matching
576 # the keywords that can be expanded in the file
577 #
578 def p4_keywords_regexp_for_type(base, type_mods):
579     if base in ("text", "unicode", "binary"):
580         kwords = None
581         if "ko" in type_mods:
582             kwords = 'Id|Header'
583         elif "k" in type_mods:
584             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
585         else:
586             return None
587         pattern = r"""
588             \$              # Starts with a dollar, followed by...
589             (%s)            # one of the keywords, followed by...
590             (:[^$\n]+)?     # possibly an old expansion, followed by...
591             \$              # another dollar
592             """ % kwords
593         return pattern
594     else:
595         return None
596
597 #
598 # Given a file, return a regexp matching the possible
599 # RCS keywords that will be expanded, or None for files
600 # with kw expansion turned off.
601 #
602 def p4_keywords_regexp_for_file(file):
603     if not os.path.exists(file):
604         return None
605     else:
606         (type_base, type_mods) = split_p4_type(p4_type(file))
607         return p4_keywords_regexp_for_type(type_base, type_mods)
608
609 def setP4ExecBit(file, mode):
610     # Reopens an already open file and changes the execute bit to match
611     # the execute bit setting in the passed in mode.
612
613     p4Type = "+x"
614
615     if not isModeExec(mode):
616         p4Type = getP4OpenedType(file)
617         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
618         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
619         if p4Type[-1] == "+":
620             p4Type = p4Type[0:-1]
621
622     p4_reopen(p4Type, file)
623
624 def getP4OpenedType(file):
625     # Returns the perforce file type for the given file.
626
627     result = p4_read_pipe(["opened", wildcard_encode(file)])
628     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
629     if match:
630         return match.group(1)
631     else:
632         die("Could not determine file type for %s (result: '%s')" % (file, result))
633
634 # Return the set of all p4 labels
635 def getP4Labels(depotPaths):
636     labels = set()
637     if not isinstance(depotPaths, list):
638         depotPaths = [depotPaths]
639
640     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
641         label = l['label']
642         labels.add(label)
643
644     return labels
645
646 # Return the set of all git tags
647 def getGitTags():
648     gitTags = set()
649     for line in read_pipe_lines(["git", "tag"]):
650         tag = line.strip()
651         gitTags.add(tag)
652     return gitTags
653
654 _diff_tree_pattern = None
655
656 def parseDiffTreeEntry(entry):
657     """Parses a single diff tree entry into its component elements.
658
659     See git-diff-tree(1) manpage for details about the format of the diff
660     output. This method returns a dictionary with the following elements:
661
662     src_mode - The mode of the source file
663     dst_mode - The mode of the destination file
664     src_sha1 - The sha1 for the source file
665     dst_sha1 - The sha1 fr the destination file
666     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
667     status_score - The score for the status (applicable for 'C' and 'R'
668                    statuses). This is None if there is no score.
669     src - The path for the source file.
670     dst - The path for the destination file. This is only present for
671           copy or renames. If it is not present, this is None.
672
673     If the pattern is not matched, None is returned."""
674
675     global _diff_tree_pattern
676     if not _diff_tree_pattern:
677         _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
678
679     match = _diff_tree_pattern.match(entry)
680     if match:
681         return {
682             'src_mode': match.group(1),
683             'dst_mode': match.group(2),
684             'src_sha1': match.group(3),
685             'dst_sha1': match.group(4),
686             'status': match.group(5),
687             'status_score': match.group(6),
688             'src': match.group(7),
689             'dst': match.group(10)
690         }
691     return None
692
693 def isModeExec(mode):
694     # Returns True if the given git mode represents an executable file,
695     # otherwise False.
696     return mode[-3:] == "755"
697
698 class P4Exception(Exception):
699     """ Base class for exceptions from the p4 client """
700     def __init__(self, exit_code):
701         self.p4ExitCode = exit_code
702
703 class P4ServerException(P4Exception):
704     """ Base class for exceptions where we get some kind of marshalled up result from the server """
705     def __init__(self, exit_code, p4_result):
706         super(P4ServerException, self).__init__(exit_code)
707         self.p4_result = p4_result
708         self.code = p4_result[0]['code']
709         self.data = p4_result[0]['data']
710
711 class P4RequestSizeException(P4ServerException):
712     """ One of the maxresults or maxscanrows errors """
713     def __init__(self, exit_code, p4_result, limit):
714         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
715         self.limit = limit
716
717 class P4CommandException(P4Exception):
718     """ Something went wrong calling p4 which means we have to give up """
719     def __init__(self, msg):
720         self.msg = msg
721
722     def __str__(self):
723         return self.msg
724
725 def isModeExecChanged(src_mode, dst_mode):
726     return isModeExec(src_mode) != isModeExec(dst_mode)
727
728 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
729         errors_as_exceptions=False):
730
731     if not isinstance(cmd, list):
732         cmd = "-G " + cmd
733         expand = True
734     else:
735         cmd = ["-G"] + cmd
736         expand = False
737
738     cmd = p4_build_cmd(cmd)
739     if verbose:
740         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
741
742     # Use a temporary file to avoid deadlocks without
743     # subprocess.communicate(), which would put another copy
744     # of stdout into memory.
745     stdin_file = None
746     if stdin is not None:
747         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
748         if not isinstance(stdin, list):
749             stdin_file.write(stdin)
750         else:
751             for i in stdin:
752                 stdin_file.write(encode_text_stream(i))
753                 stdin_file.write(b'\n')
754         stdin_file.flush()
755         stdin_file.seek(0)
756
757     p4 = subprocess.Popen(cmd,
758                           shell=expand,
759                           stdin=stdin_file,
760                           stdout=subprocess.PIPE)
761
762     result = []
763     try:
764         while True:
765             entry = marshal.load(p4.stdout)
766             if bytes is not str:
767                 # Decode unmarshalled dict to use str keys and values, except for:
768                 #   - `data` which may contain arbitrary binary data
769                 #   - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
770                 decoded_entry = {}
771                 for key, value in entry.items():
772                     key = key.decode()
773                     if isinstance(value, bytes) and not (key in ('data', 'path', 'clientFile') or key.startswith('depotFile')):
774                         value = value.decode()
775                     decoded_entry[key] = value
776                 # Parse out data if it's an error response
777                 if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
778                     decoded_entry['data'] = decoded_entry['data'].decode()
779                 entry = decoded_entry
780             if skip_info:
781                 if 'code' in entry and entry['code'] == 'info':
782                     continue
783             if cb is not None:
784                 cb(entry)
785             else:
786                 result.append(entry)
787     except EOFError:
788         pass
789     exitCode = p4.wait()
790     if exitCode != 0:
791         if errors_as_exceptions:
792             if len(result) > 0:
793                 data = result[0].get('data')
794                 if data:
795                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
796                     if not m:
797                         m = re.search('Request too large \(over (\d+)\)', data)
798
799                     if m:
800                         limit = int(m.group(1))
801                         raise P4RequestSizeException(exitCode, result, limit)
802
803                 raise P4ServerException(exitCode, result)
804             else:
805                 raise P4Exception(exitCode)
806         else:
807             entry = {}
808             entry["p4ExitCode"] = exitCode
809             result.append(entry)
810
811     return result
812
813 def p4Cmd(cmd):
814     list = p4CmdList(cmd)
815     result = {}
816     for entry in list:
817         result.update(entry)
818     return result;
819
820 def p4Where(depotPath):
821     if not depotPath.endswith("/"):
822         depotPath += "/"
823     depotPathLong = depotPath + "..."
824     outputList = p4CmdList(["where", depotPathLong])
825     output = None
826     for entry in outputList:
827         if "depotFile" in entry:
828             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
829             # The base path always ends with "/...".
830             entry_path = decode_path(entry['depotFile'])
831             if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
832                 output = entry
833                 break
834         elif "data" in entry:
835             data = entry.get("data")
836             space = data.find(" ")
837             if data[:space] == depotPath:
838                 output = entry
839                 break
840     if output == None:
841         return ""
842     if output["code"] == "error":
843         return ""
844     clientPath = ""
845     if "path" in output:
846         clientPath = decode_path(output['path'])
847     elif "data" in output:
848         data = output.get("data")
849         lastSpace = data.rfind(b" ")
850         clientPath = decode_path(data[lastSpace + 1:])
851
852     if clientPath.endswith("..."):
853         clientPath = clientPath[:-3]
854     return clientPath
855
856 def currentGitBranch():
857     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
858
859 def isValidGitDir(path):
860     return git_dir(path) != None
861
862 def parseRevision(ref):
863     return read_pipe("git rev-parse %s" % ref).strip()
864
865 def branchExists(ref):
866     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
867                      ignore_error=True)
868     return len(rev) > 0
869
870 def extractLogMessageFromGitCommit(commit):
871     logMessage = ""
872
873     ## fixme: title is first line of commit, not 1st paragraph.
874     foundTitle = False
875     for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
876        if not foundTitle:
877            if len(log) == 1:
878                foundTitle = True
879            continue
880
881        logMessage += log
882     return logMessage
883
884 def extractSettingsGitLog(log):
885     values = {}
886     for line in log.split("\n"):
887         line = line.strip()
888         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
889         if not m:
890             continue
891
892         assignments = m.group(1).split (':')
893         for a in assignments:
894             vals = a.split ('=')
895             key = vals[0].strip()
896             val = ('='.join (vals[1:])).strip()
897             if val.endswith ('\"') and val.startswith('"'):
898                 val = val[1:-1]
899
900             values[key] = val
901
902     paths = values.get("depot-paths")
903     if not paths:
904         paths = values.get("depot-path")
905     if paths:
906         values['depot-paths'] = paths.split(',')
907     return values
908
909 def gitBranchExists(branch):
910     proc = subprocess.Popen(["git", "rev-parse", branch],
911                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
912     return proc.wait() == 0;
913
914 def gitUpdateRef(ref, newvalue):
915     subprocess.check_call(["git", "update-ref", ref, newvalue])
916
917 def gitDeleteRef(ref):
918     subprocess.check_call(["git", "update-ref", "-d", ref])
919
920 _gitConfig = {}
921
922 def gitConfig(key, typeSpecifier=None):
923     if key not in _gitConfig:
924         cmd = [ "git", "config" ]
925         if typeSpecifier:
926             cmd += [ typeSpecifier ]
927         cmd += [ key ]
928         s = read_pipe(cmd, ignore_error=True)
929         _gitConfig[key] = s.strip()
930     return _gitConfig[key]
931
932 def gitConfigBool(key):
933     """Return a bool, using git config --bool.  It is True only if the
934        variable is set to true, and False if set to false or not present
935        in the config."""
936
937     if key not in _gitConfig:
938         _gitConfig[key] = gitConfig(key, '--bool') == "true"
939     return _gitConfig[key]
940
941 def gitConfigInt(key):
942     if key not in _gitConfig:
943         cmd = [ "git", "config", "--int", key ]
944         s = read_pipe(cmd, ignore_error=True)
945         v = s.strip()
946         try:
947             _gitConfig[key] = int(gitConfig(key, '--int'))
948         except ValueError:
949             _gitConfig[key] = None
950     return _gitConfig[key]
951
952 def gitConfigList(key):
953     if key not in _gitConfig:
954         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
955         _gitConfig[key] = s.strip().splitlines()
956         if _gitConfig[key] == ['']:
957             _gitConfig[key] = []
958     return _gitConfig[key]
959
960 def p4BranchesInGit(branchesAreInRemotes=True):
961     """Find all the branches whose names start with "p4/", looking
962        in remotes or heads as specified by the argument.  Return
963        a dictionary of { branch: revision } for each one found.
964        The branch names are the short names, without any
965        "p4/" prefix."""
966
967     branches = {}
968
969     cmdline = "git rev-parse --symbolic "
970     if branchesAreInRemotes:
971         cmdline += "--remotes"
972     else:
973         cmdline += "--branches"
974
975     for line in read_pipe_lines(cmdline):
976         line = line.strip()
977
978         # only import to p4/
979         if not line.startswith('p4/'):
980             continue
981         # special symbolic ref to p4/master
982         if line == "p4/HEAD":
983             continue
984
985         # strip off p4/ prefix
986         branch = line[len("p4/"):]
987
988         branches[branch] = parseRevision(line)
989
990     return branches
991
992 def branch_exists(branch):
993     """Make sure that the given ref name really exists."""
994
995     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
996     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
997     out, _ = p.communicate()
998     out = decode_text_stream(out)
999     if p.returncode:
1000         return False
1001     # expect exactly one line of output: the branch name
1002     return out.rstrip() == branch
1003
1004 def findUpstreamBranchPoint(head = "HEAD"):
1005     branches = p4BranchesInGit()
1006     # map from depot-path to branch name
1007     branchByDepotPath = {}
1008     for branch in branches.keys():
1009         tip = branches[branch]
1010         log = extractLogMessageFromGitCommit(tip)
1011         settings = extractSettingsGitLog(log)
1012         if "depot-paths" in settings:
1013             paths = ",".join(settings["depot-paths"])
1014             branchByDepotPath[paths] = "remotes/p4/" + branch
1015
1016     settings = None
1017     parent = 0
1018     while parent < 65535:
1019         commit = head + "~%s" % parent
1020         log = extractLogMessageFromGitCommit(commit)
1021         settings = extractSettingsGitLog(log)
1022         if "depot-paths" in settings:
1023             paths = ",".join(settings["depot-paths"])
1024             if paths in branchByDepotPath:
1025                 return [branchByDepotPath[paths], settings]
1026
1027         parent = parent + 1
1028
1029     return ["", settings]
1030
1031 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
1032     if not silent:
1033         print("Creating/updating branch(es) in %s based on origin branch(es)"
1034                % localRefPrefix)
1035
1036     originPrefix = "origin/p4/"
1037
1038     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
1039         line = line.strip()
1040         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
1041             continue
1042
1043         headName = line[len(originPrefix):]
1044         remoteHead = localRefPrefix + headName
1045         originHead = line
1046
1047         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
1048         if ('depot-paths' not in original
1049             or 'change' not in original):
1050             continue
1051
1052         update = False
1053         if not gitBranchExists(remoteHead):
1054             if verbose:
1055                 print("creating %s" % remoteHead)
1056             update = True
1057         else:
1058             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
1059             if 'change' in settings:
1060                 if settings['depot-paths'] == original['depot-paths']:
1061                     originP4Change = int(original['change'])
1062                     p4Change = int(settings['change'])
1063                     if originP4Change > p4Change:
1064                         print("%s (%s) is newer than %s (%s). "
1065                                "Updating p4 branch from origin."
1066                                % (originHead, originP4Change,
1067                                   remoteHead, p4Change))
1068                         update = True
1069                 else:
1070                     print("Ignoring: %s was imported from %s while "
1071                            "%s was imported from %s"
1072                            % (originHead, ','.join(original['depot-paths']),
1073                               remoteHead, ','.join(settings['depot-paths'])))
1074
1075         if update:
1076             system("git update-ref %s %s" % (remoteHead, originHead))
1077
1078 def originP4BranchesExist():
1079         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1080
1081
1082 def p4ParseNumericChangeRange(parts):
1083     changeStart = int(parts[0][1:])
1084     if parts[1] == '#head':
1085         changeEnd = p4_last_change()
1086     else:
1087         changeEnd = int(parts[1])
1088
1089     return (changeStart, changeEnd)
1090
1091 def chooseBlockSize(blockSize):
1092     if blockSize:
1093         return blockSize
1094     else:
1095         return defaultBlockSize
1096
1097 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
1098     assert depotPaths
1099
1100     # Parse the change range into start and end. Try to find integer
1101     # revision ranges as these can be broken up into blocks to avoid
1102     # hitting server-side limits (maxrows, maxscanresults). But if
1103     # that doesn't work, fall back to using the raw revision specifier
1104     # strings, without using block mode.
1105
1106     if changeRange is None or changeRange == '':
1107         changeStart = 1
1108         changeEnd = p4_last_change()
1109         block_size = chooseBlockSize(requestedBlockSize)
1110     else:
1111         parts = changeRange.split(',')
1112         assert len(parts) == 2
1113         try:
1114             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
1115             block_size = chooseBlockSize(requestedBlockSize)
1116         except ValueError:
1117             changeStart = parts[0][1:]
1118             changeEnd = parts[1]
1119             if requestedBlockSize:
1120                 die("cannot use --changes-block-size with non-numeric revisions")
1121             block_size = None
1122
1123     changes = set()
1124
1125     # Retrieve changes a block at a time, to prevent running
1126     # into a MaxResults/MaxScanRows error from the server. If
1127     # we _do_ hit one of those errors, turn down the block size
1128
1129     while True:
1130         cmd = ['changes']
1131
1132         if block_size:
1133             end = min(changeEnd, changeStart + block_size)
1134             revisionRange = "%d,%d" % (changeStart, end)
1135         else:
1136             revisionRange = "%s,%s" % (changeStart, changeEnd)
1137
1138         for p in depotPaths:
1139             cmd += ["%s...@%s" % (p, revisionRange)]
1140
1141         # fetch the changes
1142         try:
1143             result = p4CmdList(cmd, errors_as_exceptions=True)
1144         except P4RequestSizeException as e:
1145             if not block_size:
1146                 block_size = e.limit
1147             elif block_size > e.limit:
1148                 block_size = e.limit
1149             else:
1150                 block_size = max(2, block_size // 2)
1151
1152             if verbose: print("block size error, retrying with block size {0}".format(block_size))
1153             continue
1154         except P4Exception as e:
1155             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1156
1157         # Insert changes in chronological order
1158         for entry in reversed(result):
1159             if 'change' not in entry:
1160                 continue
1161             changes.add(int(entry['change']))
1162
1163         if not block_size:
1164             break
1165
1166         if end >= changeEnd:
1167             break
1168
1169         changeStart = end + 1
1170
1171     changes = sorted(changes)
1172     return changes
1173
1174 def p4PathStartsWith(path, prefix):
1175     # This method tries to remedy a potential mixed-case issue:
1176     #
1177     # If UserA adds  //depot/DirA/file1
1178     # and UserB adds //depot/dira/file2
1179     #
1180     # we may or may not have a problem. If you have core.ignorecase=true,
1181     # we treat DirA and dira as the same directory
1182     if gitConfigBool("core.ignorecase"):
1183         return path.lower().startswith(prefix.lower())
1184     return path.startswith(prefix)
1185
1186 def getClientSpec():
1187     """Look at the p4 client spec, create a View() object that contains
1188        all the mappings, and return it."""
1189
1190     specList = p4CmdList("client -o")
1191     if len(specList) != 1:
1192         die('Output from "client -o" is %d lines, expecting 1' %
1193             len(specList))
1194
1195     # dictionary of all client parameters
1196     entry = specList[0]
1197
1198     # the //client/ name
1199     client_name = entry["Client"]
1200
1201     # just the keys that start with "View"
1202     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1203
1204     # hold this new View
1205     view = View(client_name)
1206
1207     # append the lines, in order, to the view
1208     for view_num in range(len(view_keys)):
1209         k = "View%d" % view_num
1210         if k not in view_keys:
1211             die("Expected view key %s missing" % k)
1212         view.append(entry[k])
1213
1214     return view
1215
1216 def getClientRoot():
1217     """Grab the client directory."""
1218
1219     output = p4CmdList("client -o")
1220     if len(output) != 1:
1221         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1222
1223     entry = output[0]
1224     if "Root" not in entry:
1225         die('Client has no "Root"')
1226
1227     return entry["Root"]
1228
1229 #
1230 # P4 wildcards are not allowed in filenames.  P4 complains
1231 # if you simply add them, but you can force it with "-f", in
1232 # which case it translates them into %xx encoding internally.
1233 #
1234 def wildcard_decode(path):
1235     # Search for and fix just these four characters.  Do % last so
1236     # that fixing it does not inadvertently create new %-escapes.
1237     # Cannot have * in a filename in windows; untested as to
1238     # what p4 would do in such a case.
1239     if not platform.system() == "Windows":
1240         path = path.replace("%2A", "*")
1241     path = path.replace("%23", "#") \
1242                .replace("%40", "@") \
1243                .replace("%25", "%")
1244     return path
1245
1246 def wildcard_encode(path):
1247     # do % first to avoid double-encoding the %s introduced here
1248     path = path.replace("%", "%25") \
1249                .replace("*", "%2A") \
1250                .replace("#", "%23") \
1251                .replace("@", "%40")
1252     return path
1253
1254 def wildcard_present(path):
1255     m = re.search("[*#@%]", path)
1256     return m is not None
1257
1258 class LargeFileSystem(object):
1259     """Base class for large file system support."""
1260
1261     def __init__(self, writeToGitStream):
1262         self.largeFiles = set()
1263         self.writeToGitStream = writeToGitStream
1264
1265     def generatePointer(self, cloneDestination, contentFile):
1266         """Return the content of a pointer file that is stored in Git instead of
1267            the actual content."""
1268         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1269
1270     def pushFile(self, localLargeFile):
1271         """Push the actual content which is not stored in the Git repository to
1272            a server."""
1273         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1274
1275     def hasLargeFileExtension(self, relPath):
1276         return functools.reduce(
1277             lambda a, b: a or b,
1278             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1279             False
1280         )
1281
1282     def generateTempFile(self, contents):
1283         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1284         for d in contents:
1285             contentFile.write(d)
1286         contentFile.close()
1287         return contentFile.name
1288
1289     def exceedsLargeFileThreshold(self, relPath, contents):
1290         if gitConfigInt('git-p4.largeFileThreshold'):
1291             contentsSize = sum(len(d) for d in contents)
1292             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1293                 return True
1294         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1295             contentsSize = sum(len(d) for d in contents)
1296             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1297                 return False
1298             contentTempFile = self.generateTempFile(contents)
1299             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1300             with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1301                 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1302                 compressedContentsSize = zf.infolist()[0].compress_size
1303             os.remove(contentTempFile)
1304             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1305                 return True
1306         return False
1307
1308     def addLargeFile(self, relPath):
1309         self.largeFiles.add(relPath)
1310
1311     def removeLargeFile(self, relPath):
1312         self.largeFiles.remove(relPath)
1313
1314     def isLargeFile(self, relPath):
1315         return relPath in self.largeFiles
1316
1317     def processContent(self, git_mode, relPath, contents):
1318         """Processes the content of git fast import. This method decides if a
1319            file is stored in the large file system and handles all necessary
1320            steps."""
1321         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1322             contentTempFile = self.generateTempFile(contents)
1323             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1324             if pointer_git_mode:
1325                 git_mode = pointer_git_mode
1326             if localLargeFile:
1327                 # Move temp file to final location in large file system
1328                 largeFileDir = os.path.dirname(localLargeFile)
1329                 if not os.path.isdir(largeFileDir):
1330                     os.makedirs(largeFileDir)
1331                 shutil.move(contentTempFile, localLargeFile)
1332                 self.addLargeFile(relPath)
1333                 if gitConfigBool('git-p4.largeFilePush'):
1334                     self.pushFile(localLargeFile)
1335                 if verbose:
1336                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1337         return (git_mode, contents)
1338
1339 class MockLFS(LargeFileSystem):
1340     """Mock large file system for testing."""
1341
1342     def generatePointer(self, contentFile):
1343         """The pointer content is the original content prefixed with "pointer-".
1344            The local filename of the large file storage is derived from the file content.
1345            """
1346         with open(contentFile, 'r') as f:
1347             content = next(f)
1348             gitMode = '100644'
1349             pointerContents = 'pointer-' + content
1350             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1351             return (gitMode, pointerContents, localLargeFile)
1352
1353     def pushFile(self, localLargeFile):
1354         """The remote filename of the large file storage is the same as the local
1355            one but in a different directory.
1356            """
1357         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1358         if not os.path.exists(remotePath):
1359             os.makedirs(remotePath)
1360         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1361
1362 class GitLFS(LargeFileSystem):
1363     """Git LFS as backend for the git-p4 large file system.
1364        See https://git-lfs.github.com/ for details."""
1365
1366     def __init__(self, *args):
1367         LargeFileSystem.__init__(self, *args)
1368         self.baseGitAttributes = []
1369
1370     def generatePointer(self, contentFile):
1371         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1372            mode and content which is stored in the Git repository instead of
1373            the actual content. Return also the new location of the actual
1374            content.
1375            """
1376         if os.path.getsize(contentFile) == 0:
1377             return (None, '', None)
1378
1379         pointerProcess = subprocess.Popen(
1380             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1381             stdout=subprocess.PIPE
1382         )
1383         pointerFile = decode_text_stream(pointerProcess.stdout.read())
1384         if pointerProcess.wait():
1385             os.remove(contentFile)
1386             die('git-lfs pointer command failed. Did you install the extension?')
1387
1388         # Git LFS removed the preamble in the output of the 'pointer' command
1389         # starting from version 1.2.0. Check for the preamble here to support
1390         # earlier versions.
1391         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1392         if pointerFile.startswith('Git LFS pointer for'):
1393             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1394
1395         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1396         # if someone use external lfs.storage ( not in local repo git )
1397         lfs_path = gitConfig('lfs.storage')
1398         if not lfs_path:
1399             lfs_path = 'lfs'
1400         if not os.path.isabs(lfs_path):
1401             lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1402         localLargeFile = os.path.join(
1403             lfs_path,
1404             'objects', oid[:2], oid[2:4],
1405             oid,
1406         )
1407         # LFS Spec states that pointer files should not have the executable bit set.
1408         gitMode = '100644'
1409         return (gitMode, pointerFile, localLargeFile)
1410
1411     def pushFile(self, localLargeFile):
1412         uploadProcess = subprocess.Popen(
1413             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1414         )
1415         if uploadProcess.wait():
1416             die('git-lfs push command failed. Did you define a remote?')
1417
1418     def generateGitAttributes(self):
1419         return (
1420             self.baseGitAttributes +
1421             [
1422                 '\n',
1423                 '#\n',
1424                 '# Git LFS (see https://git-lfs.github.com/)\n',
1425                 '#\n',
1426             ] +
1427             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1428                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1429             ] +
1430             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1431                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1432             ]
1433         )
1434
1435     def addLargeFile(self, relPath):
1436         LargeFileSystem.addLargeFile(self, relPath)
1437         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1438
1439     def removeLargeFile(self, relPath):
1440         LargeFileSystem.removeLargeFile(self, relPath)
1441         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1442
1443     def processContent(self, git_mode, relPath, contents):
1444         if relPath == '.gitattributes':
1445             self.baseGitAttributes = contents
1446             return (git_mode, self.generateGitAttributes())
1447         else:
1448             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1449
1450 class Command:
1451     delete_actions = ( "delete", "move/delete", "purge" )
1452     add_actions = ( "add", "branch", "move/add" )
1453
1454     def __init__(self):
1455         self.usage = "usage: %prog [options]"
1456         self.needsGit = True
1457         self.verbose = False
1458
1459     # This is required for the "append" update_shelve action
1460     def ensure_value(self, attr, value):
1461         if not hasattr(self, attr) or getattr(self, attr) is None:
1462             setattr(self, attr, value)
1463         return getattr(self, attr)
1464
1465 class P4UserMap:
1466     def __init__(self):
1467         self.userMapFromPerforceServer = False
1468         self.myP4UserId = None
1469
1470     def p4UserId(self):
1471         if self.myP4UserId:
1472             return self.myP4UserId
1473
1474         results = p4CmdList("user -o")
1475         for r in results:
1476             if 'User' in r:
1477                 self.myP4UserId = r['User']
1478                 return r['User']
1479         die("Could not find your p4 user id")
1480
1481     def p4UserIsMe(self, p4User):
1482         # return True if the given p4 user is actually me
1483         me = self.p4UserId()
1484         if not p4User or p4User != me:
1485             return False
1486         else:
1487             return True
1488
1489     def getUserCacheFilename(self):
1490         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1491         return home + "/.gitp4-usercache.txt"
1492
1493     def getUserMapFromPerforceServer(self):
1494         if self.userMapFromPerforceServer:
1495             return
1496         self.users = {}
1497         self.emails = {}
1498
1499         for output in p4CmdList("users"):
1500             if "User" not in output:
1501                 continue
1502             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1503             self.emails[output["Email"]] = output["User"]
1504
1505         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1506         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1507             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1508             if mapUser and len(mapUser[0]) == 3:
1509                 user = mapUser[0][0]
1510                 fullname = mapUser[0][1]
1511                 email = mapUser[0][2]
1512                 self.users[user] = fullname + " <" + email + ">"
1513                 self.emails[email] = user
1514
1515         s = ''
1516         for (key, val) in self.users.items():
1517             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1518
1519         open(self.getUserCacheFilename(), 'w').write(s)
1520         self.userMapFromPerforceServer = True
1521
1522     def loadUserMapFromCache(self):
1523         self.users = {}
1524         self.userMapFromPerforceServer = False
1525         try:
1526             cache = open(self.getUserCacheFilename(), 'r')
1527             lines = cache.readlines()
1528             cache.close()
1529             for line in lines:
1530                 entry = line.strip().split("\t")
1531                 self.users[entry[0]] = entry[1]
1532         except IOError:
1533             self.getUserMapFromPerforceServer()
1534
1535 class P4Debug(Command):
1536     def __init__(self):
1537         Command.__init__(self)
1538         self.options = []
1539         self.description = "A tool to debug the output of p4 -G."
1540         self.needsGit = False
1541
1542     def run(self, args):
1543         j = 0
1544         for output in p4CmdList(args):
1545             print('Element: %d' % j)
1546             j += 1
1547             print(output)
1548         return True
1549
1550 class P4RollBack(Command):
1551     def __init__(self):
1552         Command.__init__(self)
1553         self.options = [
1554             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1555         ]
1556         self.description = "A tool to debug the multi-branch import. Don't use :)"
1557         self.rollbackLocalBranches = False
1558
1559     def run(self, args):
1560         if len(args) != 1:
1561             return False
1562         maxChange = int(args[0])
1563
1564         if "p4ExitCode" in p4Cmd("changes -m 1"):
1565             die("Problems executing p4");
1566
1567         if self.rollbackLocalBranches:
1568             refPrefix = "refs/heads/"
1569             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1570         else:
1571             refPrefix = "refs/remotes/"
1572             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1573
1574         for line in lines:
1575             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1576                 line = line.strip()
1577                 ref = refPrefix + line
1578                 log = extractLogMessageFromGitCommit(ref)
1579                 settings = extractSettingsGitLog(log)
1580
1581                 depotPaths = settings['depot-paths']
1582                 change = settings['change']
1583
1584                 changed = False
1585
1586                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1587                                                            for p in depotPaths]))) == 0:
1588                     print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1589                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1590                     continue
1591
1592                 while change and int(change) > maxChange:
1593                     changed = True
1594                     if self.verbose:
1595                         print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1596                     system("git update-ref %s \"%s^\"" % (ref, ref))
1597                     log = extractLogMessageFromGitCommit(ref)
1598                     settings =  extractSettingsGitLog(log)
1599
1600
1601                     depotPaths = settings['depot-paths']
1602                     change = settings['change']
1603
1604                 if changed:
1605                     print("%s rewound to %s" % (ref, change))
1606
1607         return True
1608
1609 class P4Submit(Command, P4UserMap):
1610
1611     conflict_behavior_choices = ("ask", "skip", "quit")
1612
1613     def __init__(self):
1614         Command.__init__(self)
1615         P4UserMap.__init__(self)
1616         self.options = [
1617                 optparse.make_option("--origin", dest="origin"),
1618                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1619                 # preserve the user, requires relevant p4 permissions
1620                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1621                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1622                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1623                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1624                 optparse.make_option("--conflict", dest="conflict_behavior",
1625                                      choices=self.conflict_behavior_choices),
1626                 optparse.make_option("--branch", dest="branch"),
1627                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1628                                      help="Shelve instead of submit. Shelved files are reverted, "
1629                                      "restoring the workspace to the state before the shelve"),
1630                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1631                                      metavar="CHANGELIST",
1632                                      help="update an existing shelved changelist, implies --shelve, "
1633                                            "repeat in-order for multiple shelved changelists"),
1634                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1635                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1636                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1637                                      help="Disable rebase after submit is completed. Can be useful if you "
1638                                      "work from a local git branch that is not master"),
1639                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1640                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1641                 optparse.make_option("--no-verify", dest="no_verify", action="store_true",
1642                                      help="Bypass p4-pre-submit and p4-changelist hooks"),
1643         ]
1644         self.description = """Submit changes from git to the perforce depot.\n
1645     The `p4-pre-submit` hook is executed if it exists and is executable. It
1646     can be bypassed with the `--no-verify` command line option. The hook takes
1647     no parameters and nothing from standard input. Exiting with a non-zero status
1648     from this script prevents `git-p4 submit` from launching.
1649
1650     One usage scenario is to run unit tests in the hook.
1651
1652     The `p4-prepare-changelist` hook is executed right after preparing the default
1653     changelist message and before the editor is started. It takes one parameter,
1654     the name of the file that contains the changelist text. Exiting with a non-zero
1655     status from the script will abort the process.
1656
1657     The purpose of the hook is to edit the message file in place, and it is not
1658     supressed by the `--no-verify` option. This hook is called even if
1659     `--prepare-p4-only` is set.
1660
1661     The `p4-changelist` hook is executed after the changelist message has been
1662     edited by the user. It can be bypassed with the `--no-verify` option. It
1663     takes a single parameter, the name of the file that holds the proposed
1664     changelist text. Exiting with a non-zero status causes the command to abort.
1665
1666     The hook is allowed to edit the changelist file and can be used to normalize
1667     the text into some project standard format. It can also be used to refuse the
1668     Submit after inspect the message file.
1669
1670     The `p4-post-changelist` hook is invoked after the submit has successfully
1671     occurred in P4. It takes no parameters and is meant primarily for notification
1672     and cannot affect the outcome of the git p4 submit action.
1673     """
1674
1675         self.usage += " [name of git branch to submit into perforce depot]"
1676         self.origin = ""
1677         self.detectRenames = False
1678         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1679         self.dry_run = False
1680         self.shelve = False
1681         self.update_shelve = list()
1682         self.commit = ""
1683         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1684         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1685         self.prepare_p4_only = False
1686         self.conflict_behavior = None
1687         self.isWindows = (platform.system() == "Windows")
1688         self.exportLabels = False
1689         self.p4HasMoveCommand = p4_has_move_command()
1690         self.branch = None
1691         self.no_verify = False
1692
1693         if gitConfig('git-p4.largeFileSystem'):
1694             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1695
1696     def check(self):
1697         if len(p4CmdList("opened ...")) > 0:
1698             die("You have files opened with perforce! Close them before starting the sync.")
1699
1700     def separate_jobs_from_description(self, message):
1701         """Extract and return a possible Jobs field in the commit
1702            message.  It goes into a separate section in the p4 change
1703            specification.
1704
1705            A jobs line starts with "Jobs:" and looks like a new field
1706            in a form.  Values are white-space separated on the same
1707            line or on following lines that start with a tab.
1708
1709            This does not parse and extract the full git commit message
1710            like a p4 form.  It just sees the Jobs: line as a marker
1711            to pass everything from then on directly into the p4 form,
1712            but outside the description section.
1713
1714            Return a tuple (stripped log message, jobs string)."""
1715
1716         m = re.search(r'^Jobs:', message, re.MULTILINE)
1717         if m is None:
1718             return (message, None)
1719
1720         jobtext = message[m.start():]
1721         stripped_message = message[:m.start()].rstrip()
1722         return (stripped_message, jobtext)
1723
1724     def prepareLogMessage(self, template, message, jobs):
1725         """Edits the template returned from "p4 change -o" to insert
1726            the message in the Description field, and the jobs text in
1727            the Jobs field."""
1728         result = ""
1729
1730         inDescriptionSection = False
1731
1732         for line in template.split("\n"):
1733             if line.startswith("#"):
1734                 result += line + "\n"
1735                 continue
1736
1737             if inDescriptionSection:
1738                 if line.startswith("Files:") or line.startswith("Jobs:"):
1739                     inDescriptionSection = False
1740                     # insert Jobs section
1741                     if jobs:
1742                         result += jobs + "\n"
1743                 else:
1744                     continue
1745             else:
1746                 if line.startswith("Description:"):
1747                     inDescriptionSection = True
1748                     line += "\n"
1749                     for messageLine in message.split("\n"):
1750                         line += "\t" + messageLine + "\n"
1751
1752             result += line + "\n"
1753
1754         return result
1755
1756     def patchRCSKeywords(self, file, pattern):
1757         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1758         (handle, outFileName) = tempfile.mkstemp(dir='.')
1759         try:
1760             outFile = os.fdopen(handle, "w+")
1761             inFile = open(file, "r")
1762             regexp = re.compile(pattern, re.VERBOSE)
1763             for line in inFile.readlines():
1764                 line = regexp.sub(r'$\1$', line)
1765                 outFile.write(line)
1766             inFile.close()
1767             outFile.close()
1768             # Forcibly overwrite the original file
1769             os.unlink(file)
1770             shutil.move(outFileName, file)
1771         except:
1772             # cleanup our temporary file
1773             os.unlink(outFileName)
1774             print("Failed to strip RCS keywords in %s" % file)
1775             raise
1776
1777         print("Patched up RCS keywords in %s" % file)
1778
1779     def p4UserForCommit(self,id):
1780         # Return the tuple (perforce user,git email) for a given git commit id
1781         self.getUserMapFromPerforceServer()
1782         gitEmail = read_pipe(["git", "log", "--max-count=1",
1783                               "--format=%ae", id])
1784         gitEmail = gitEmail.strip()
1785         if gitEmail not in self.emails:
1786             return (None,gitEmail)
1787         else:
1788             return (self.emails[gitEmail],gitEmail)
1789
1790     def checkValidP4Users(self,commits):
1791         # check if any git authors cannot be mapped to p4 users
1792         for id in commits:
1793             (user,email) = self.p4UserForCommit(id)
1794             if not user:
1795                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1796                 if gitConfigBool("git-p4.allowMissingP4Users"):
1797                     print("%s" % msg)
1798                 else:
1799                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1800
1801     def lastP4Changelist(self):
1802         # Get back the last changelist number submitted in this client spec. This
1803         # then gets used to patch up the username in the change. If the same
1804         # client spec is being used by multiple processes then this might go
1805         # wrong.
1806         results = p4CmdList("client -o")        # find the current client
1807         client = None
1808         for r in results:
1809             if 'Client' in r:
1810                 client = r['Client']
1811                 break
1812         if not client:
1813             die("could not get client spec")
1814         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1815         for r in results:
1816             if 'change' in r:
1817                 return r['change']
1818         die("Could not get changelist number for last submit - cannot patch up user details")
1819
1820     def modifyChangelistUser(self, changelist, newUser):
1821         # fixup the user field of a changelist after it has been submitted.
1822         changes = p4CmdList("change -o %s" % changelist)
1823         if len(changes) != 1:
1824             die("Bad output from p4 change modifying %s to user %s" %
1825                 (changelist, newUser))
1826
1827         c = changes[0]
1828         if c['User'] == newUser: return   # nothing to do
1829         c['User'] = newUser
1830         # p4 does not understand format version 3 and above
1831         input = marshal.dumps(c, 2)
1832
1833         result = p4CmdList("change -f -i", stdin=input)
1834         for r in result:
1835             if 'code' in r:
1836                 if r['code'] == 'error':
1837                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1838             if 'data' in r:
1839                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1840                 return
1841         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1842
1843     def canChangeChangelists(self):
1844         # check to see if we have p4 admin or super-user permissions, either of
1845         # which are required to modify changelists.
1846         results = p4CmdList(["protects", self.depotPath])
1847         for r in results:
1848             if 'perm' in r:
1849                 if r['perm'] == 'admin':
1850                     return 1
1851                 if r['perm'] == 'super':
1852                     return 1
1853         return 0
1854
1855     def prepareSubmitTemplate(self, changelist=None):
1856         """Run "p4 change -o" to grab a change specification template.
1857            This does not use "p4 -G", as it is nice to keep the submission
1858            template in original order, since a human might edit it.
1859
1860            Remove lines in the Files section that show changes to files
1861            outside the depot path we're committing into."""
1862
1863         [upstream, settings] = findUpstreamBranchPoint()
1864
1865         template = """\
1866 # A Perforce Change Specification.
1867 #
1868 #  Change:      The change number. 'new' on a new changelist.
1869 #  Date:        The date this specification was last modified.
1870 #  Client:      The client on which the changelist was created.  Read-only.
1871 #  User:        The user who created the changelist.
1872 #  Status:      Either 'pending' or 'submitted'. Read-only.
1873 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1874 #  Description: Comments about the changelist.  Required.
1875 #  Jobs:        What opened jobs are to be closed by this changelist.
1876 #               You may delete jobs from this list.  (New changelists only.)
1877 #  Files:       What opened files from the default changelist are to be added
1878 #               to this changelist.  You may delete files from this list.
1879 #               (New changelists only.)
1880 """
1881         files_list = []
1882         inFilesSection = False
1883         change_entry = None
1884         args = ['change', '-o']
1885         if changelist:
1886             args.append(str(changelist))
1887         for entry in p4CmdList(args):
1888             if 'code' not in entry:
1889                 continue
1890             if entry['code'] == 'stat':
1891                 change_entry = entry
1892                 break
1893         if not change_entry:
1894             die('Failed to decode output of p4 change -o')
1895         for key, value in change_entry.items():
1896             if key.startswith('File'):
1897                 if 'depot-paths' in settings:
1898                     if not [p for p in settings['depot-paths']
1899                             if p4PathStartsWith(value, p)]:
1900                         continue
1901                 else:
1902                     if not p4PathStartsWith(value, self.depotPath):
1903                         continue
1904                 files_list.append(value)
1905                 continue
1906         # Output in the order expected by prepareLogMessage
1907         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1908             if key not in change_entry:
1909                 continue
1910             template += '\n'
1911             template += key + ':'
1912             if key == 'Description':
1913                 template += '\n'
1914             for field_line in change_entry[key].splitlines():
1915                 template += '\t'+field_line+'\n'
1916         if len(files_list) > 0:
1917             template += '\n'
1918             template += 'Files:\n'
1919         for path in files_list:
1920             template += '\t'+path+'\n'
1921         return template
1922
1923     def edit_template(self, template_file):
1924         """Invoke the editor to let the user change the submission
1925            message.  Return true if okay to continue with the submit."""
1926
1927         # if configured to skip the editing part, just submit
1928         if gitConfigBool("git-p4.skipSubmitEdit"):
1929             return True
1930
1931         # look at the modification time, to check later if the user saved
1932         # the file
1933         mtime = os.stat(template_file).st_mtime
1934
1935         # invoke the editor
1936         if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1937             editor = os.environ.get("P4EDITOR")
1938         else:
1939             editor = read_pipe("git var GIT_EDITOR").strip()
1940         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1941
1942         # If the file was not saved, prompt to see if this patch should
1943         # be skipped.  But skip this verification step if configured so.
1944         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1945             return True
1946
1947         # modification time updated means user saved the file
1948         if os.stat(template_file).st_mtime > mtime:
1949             return True
1950
1951         response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1952         if response == 'y':
1953             return True
1954         if response == 'n':
1955             return False
1956
1957     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1958         # diff
1959         if "P4DIFF" in os.environ:
1960             del(os.environ["P4DIFF"])
1961         diff = ""
1962         for editedFile in editedFiles:
1963             diff += p4_read_pipe(['diff', '-du',
1964                                   wildcard_encode(editedFile)])
1965
1966         # new file diff
1967         newdiff = ""
1968         for newFile in filesToAdd:
1969             newdiff += "==== new file ====\n"
1970             newdiff += "--- /dev/null\n"
1971             newdiff += "+++ %s\n" % newFile
1972
1973             is_link = os.path.islink(newFile)
1974             expect_link = newFile in symlinks
1975
1976             if is_link and expect_link:
1977                 newdiff += "+%s\n" % os.readlink(newFile)
1978             else:
1979                 f = open(newFile, "r")
1980                 for line in f.readlines():
1981                     newdiff += "+" + line
1982                 f.close()
1983
1984         return (diff + newdiff).replace('\r\n', '\n')
1985
1986     def applyCommit(self, id):
1987         """Apply one commit, return True if it succeeded."""
1988
1989         print("Applying", read_pipe(["git", "show", "-s",
1990                                      "--format=format:%h %s", id]))
1991
1992         (p4User, gitEmail) = self.p4UserForCommit(id)
1993
1994         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1995         filesToAdd = set()
1996         filesToChangeType = set()
1997         filesToDelete = set()
1998         editedFiles = set()
1999         pureRenameCopy = set()
2000         symlinks = set()
2001         filesToChangeExecBit = {}
2002         all_files = list()
2003
2004         for line in diff:
2005             diff = parseDiffTreeEntry(line)
2006             modifier = diff['status']
2007             path = diff['src']
2008             all_files.append(path)
2009
2010             if modifier == "M":
2011                 p4_edit(path)
2012                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2013                     filesToChangeExecBit[path] = diff['dst_mode']
2014                 editedFiles.add(path)
2015             elif modifier == "A":
2016                 filesToAdd.add(path)
2017                 filesToChangeExecBit[path] = diff['dst_mode']
2018                 if path in filesToDelete:
2019                     filesToDelete.remove(path)
2020
2021                 dst_mode = int(diff['dst_mode'], 8)
2022                 if dst_mode == 0o120000:
2023                     symlinks.add(path)
2024
2025             elif modifier == "D":
2026                 filesToDelete.add(path)
2027                 if path in filesToAdd:
2028                     filesToAdd.remove(path)
2029             elif modifier == "C":
2030                 src, dest = diff['src'], diff['dst']
2031                 all_files.append(dest)
2032                 p4_integrate(src, dest)
2033                 pureRenameCopy.add(dest)
2034                 if diff['src_sha1'] != diff['dst_sha1']:
2035                     p4_edit(dest)
2036                     pureRenameCopy.discard(dest)
2037                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2038                     p4_edit(dest)
2039                     pureRenameCopy.discard(dest)
2040                     filesToChangeExecBit[dest] = diff['dst_mode']
2041                 if self.isWindows:
2042                     # turn off read-only attribute
2043                     os.chmod(dest, stat.S_IWRITE)
2044                 os.unlink(dest)
2045                 editedFiles.add(dest)
2046             elif modifier == "R":
2047                 src, dest = diff['src'], diff['dst']
2048                 all_files.append(dest)
2049                 if self.p4HasMoveCommand:
2050                     p4_edit(src)        # src must be open before move
2051                     p4_move(src, dest)  # opens for (move/delete, move/add)
2052                 else:
2053                     p4_integrate(src, dest)
2054                     if diff['src_sha1'] != diff['dst_sha1']:
2055                         p4_edit(dest)
2056                     else:
2057                         pureRenameCopy.add(dest)
2058                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2059                     if not self.p4HasMoveCommand:
2060                         p4_edit(dest)   # with move: already open, writable
2061                     filesToChangeExecBit[dest] = diff['dst_mode']
2062                 if not self.p4HasMoveCommand:
2063                     if self.isWindows:
2064                         os.chmod(dest, stat.S_IWRITE)
2065                     os.unlink(dest)
2066                     filesToDelete.add(src)
2067                 editedFiles.add(dest)
2068             elif modifier == "T":
2069                 filesToChangeType.add(path)
2070             else:
2071                 die("unknown modifier %s for %s" % (modifier, path))
2072
2073         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
2074         patchcmd = diffcmd + " | git apply "
2075         tryPatchCmd = patchcmd + "--check -"
2076         applyPatchCmd = patchcmd + "--check --apply -"
2077         patch_succeeded = True
2078
2079         if verbose:
2080             print("TryPatch: %s" % tryPatchCmd)
2081
2082         if os.system(tryPatchCmd) != 0:
2083             fixed_rcs_keywords = False
2084             patch_succeeded = False
2085             print("Unfortunately applying the change failed!")
2086
2087             # Patch failed, maybe it's just RCS keyword woes. Look through
2088             # the patch to see if that's possible.
2089             if gitConfigBool("git-p4.attemptRCSCleanup"):
2090                 file = None
2091                 pattern = None
2092                 kwfiles = {}
2093                 for file in editedFiles | filesToDelete:
2094                     # did this file's delta contain RCS keywords?
2095                     pattern = p4_keywords_regexp_for_file(file)
2096
2097                     if pattern:
2098                         # this file is a possibility...look for RCS keywords.
2099                         regexp = re.compile(pattern, re.VERBOSE)
2100                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
2101                             if regexp.search(line):
2102                                 if verbose:
2103                                     print("got keyword match on %s in %s in %s" % (pattern, line, file))
2104                                 kwfiles[file] = pattern
2105                                 break
2106
2107                 for file in kwfiles:
2108                     if verbose:
2109                         print("zapping %s with %s" % (line,pattern))
2110                     # File is being deleted, so not open in p4.  Must
2111                     # disable the read-only bit on windows.
2112                     if self.isWindows and file not in editedFiles:
2113                         os.chmod(file, stat.S_IWRITE)
2114                     self.patchRCSKeywords(file, kwfiles[file])
2115                     fixed_rcs_keywords = True
2116
2117             if fixed_rcs_keywords:
2118                 print("Retrying the patch with RCS keywords cleaned up")
2119                 if os.system(tryPatchCmd) == 0:
2120                     patch_succeeded = True
2121                     print("Patch succeesed this time with RCS keywords cleaned")
2122
2123         if not patch_succeeded:
2124             for f in editedFiles:
2125                 p4_revert(f)
2126             return False
2127
2128         #
2129         # Apply the patch for real, and do add/delete/+x handling.
2130         #
2131         system(applyPatchCmd)
2132
2133         for f in filesToChangeType:
2134             p4_edit(f, "-t", "auto")
2135         for f in filesToAdd:
2136             p4_add(f)
2137         for f in filesToDelete:
2138             p4_revert(f)
2139             p4_delete(f)
2140
2141         # Set/clear executable bits
2142         for f in filesToChangeExecBit.keys():
2143             mode = filesToChangeExecBit[f]
2144             setP4ExecBit(f, mode)
2145
2146         update_shelve = 0
2147         if len(self.update_shelve) > 0:
2148             update_shelve = self.update_shelve.pop(0)
2149             p4_reopen_in_change(update_shelve, all_files)
2150
2151         #
2152         # Build p4 change description, starting with the contents
2153         # of the git commit message.
2154         #
2155         logMessage = extractLogMessageFromGitCommit(id)
2156         logMessage = logMessage.strip()
2157         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
2158
2159         template = self.prepareSubmitTemplate(update_shelve)
2160         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
2161
2162         if self.preserveUser:
2163            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
2164
2165         if self.checkAuthorship and not self.p4UserIsMe(p4User):
2166             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2167             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2168             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2169
2170         separatorLine = "######## everything below this line is just the diff #######\n"
2171         if not self.prepare_p4_only:
2172             submitTemplate += separatorLine
2173             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2174
2175         (handle, fileName) = tempfile.mkstemp()
2176         tmpFile = os.fdopen(handle, "w+b")
2177         if self.isWindows:
2178             submitTemplate = submitTemplate.replace("\n", "\r\n")
2179         tmpFile.write(encode_text_stream(submitTemplate))
2180         tmpFile.close()
2181
2182         submitted = False
2183
2184         try:
2185             # Allow the hook to edit the changelist text before presenting it
2186             # to the user.
2187             if not run_git_hook("p4-prepare-changelist", [fileName]):
2188                 return False
2189
2190             if self.prepare_p4_only:
2191                 #
2192                 # Leave the p4 tree prepared, and the submit template around
2193                 # and let the user decide what to do next
2194                 #
2195                 submitted = True
2196                 print("")
2197                 print("P4 workspace prepared for submission.")
2198                 print("To submit or revert, go to client workspace")
2199                 print("  " + self.clientPath)
2200                 print("")
2201                 print("To submit, use \"p4 submit\" to write a new description,")
2202                 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2203                       " \"git p4\"." % fileName)
2204                 print("You can delete the file \"%s\" when finished." % fileName)
2205
2206                 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2207                     print("To preserve change ownership by user %s, you must\n" \
2208                           "do \"p4 change -f <change>\" after submitting and\n" \
2209                           "edit the User field.")
2210                 if pureRenameCopy:
2211                     print("After submitting, renamed files must be re-synced.")
2212                     print("Invoke \"p4 sync -f\" on each of these files:")
2213                     for f in pureRenameCopy:
2214                         print("  " + f)
2215
2216                 print("")
2217                 print("To revert the changes, use \"p4 revert ...\", and delete")
2218                 print("the submit template file \"%s\"" % fileName)
2219                 if filesToAdd:
2220                     print("Since the commit adds new files, they must be deleted:")
2221                     for f in filesToAdd:
2222                         print("  " + f)
2223                 print("")
2224                 sys.stdout.flush()
2225                 return True
2226
2227             if self.edit_template(fileName):
2228                 if not self.no_verify:
2229                     if not run_git_hook("p4-changelist", [fileName]):
2230                         print("The p4-changelist hook failed.")
2231                         sys.stdout.flush()
2232                         return False
2233
2234                 # read the edited message and submit
2235                 tmpFile = open(fileName, "rb")
2236                 message = decode_text_stream(tmpFile.read())
2237                 tmpFile.close()
2238                 if self.isWindows:
2239                     message = message.replace("\r\n", "\n")
2240                 if message.find(separatorLine) != -1:
2241                     submitTemplate = message[:message.index(separatorLine)]
2242                 else:
2243                     submitTemplate = message
2244
2245                 if len(submitTemplate.strip()) == 0:
2246                     print("Changelist is empty, aborting this changelist.")
2247                     sys.stdout.flush()
2248                     return False
2249
2250                 if update_shelve:
2251                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2252                 elif self.shelve:
2253                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2254                 else:
2255                     p4_write_pipe(['submit', '-i'], submitTemplate)
2256                     # The rename/copy happened by applying a patch that created a
2257                     # new file.  This leaves it writable, which confuses p4.
2258                     for f in pureRenameCopy:
2259                         p4_sync(f, "-f")
2260
2261                 if self.preserveUser:
2262                     if p4User:
2263                         # Get last changelist number. Cannot easily get it from
2264                         # the submit command output as the output is
2265                         # unmarshalled.
2266                         changelist = self.lastP4Changelist()
2267                         self.modifyChangelistUser(changelist, p4User)
2268
2269                 submitted = True
2270
2271                 run_git_hook("p4-post-changelist")
2272         finally:
2273             # Revert changes if we skip this patch
2274             if not submitted or self.shelve:
2275                 if self.shelve:
2276                     print ("Reverting shelved files.")
2277                 else:
2278                     print ("Submission cancelled, undoing p4 changes.")
2279                 sys.stdout.flush()
2280                 for f in editedFiles | filesToDelete:
2281                     p4_revert(f)
2282                 for f in filesToAdd:
2283                     p4_revert(f)
2284                     os.remove(f)
2285
2286             if not self.prepare_p4_only:
2287                 os.remove(fileName)
2288         return submitted
2289
2290     # Export git tags as p4 labels. Create a p4 label and then tag
2291     # with that.
2292     def exportGitTags(self, gitTags):
2293         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2294         if len(validLabelRegexp) == 0:
2295             validLabelRegexp = defaultLabelRegexp
2296         m = re.compile(validLabelRegexp)
2297
2298         for name in gitTags:
2299
2300             if not m.match(name):
2301                 if verbose:
2302                     print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2303                 continue
2304
2305             # Get the p4 commit this corresponds to
2306             logMessage = extractLogMessageFromGitCommit(name)
2307             values = extractSettingsGitLog(logMessage)
2308
2309             if 'change' not in values:
2310                 # a tag pointing to something not sent to p4; ignore
2311                 if verbose:
2312                     print("git tag %s does not give a p4 commit" % name)
2313                 continue
2314             else:
2315                 changelist = values['change']
2316
2317             # Get the tag details.
2318             inHeader = True
2319             isAnnotated = False
2320             body = []
2321             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2322                 l = l.strip()
2323                 if inHeader:
2324                     if re.match(r'tag\s+', l):
2325                         isAnnotated = True
2326                     elif re.match(r'\s*$', l):
2327                         inHeader = False
2328                         continue
2329                 else:
2330                     body.append(l)
2331
2332             if not isAnnotated:
2333                 body = ["lightweight tag imported by git p4\n"]
2334
2335             # Create the label - use the same view as the client spec we are using
2336             clientSpec = getClientSpec()
2337
2338             labelTemplate  = "Label: %s\n" % name
2339             labelTemplate += "Description:\n"
2340             for b in body:
2341                 labelTemplate += "\t" + b + "\n"
2342             labelTemplate += "View:\n"
2343             for depot_side in clientSpec.mappings:
2344                 labelTemplate += "\t%s\n" % depot_side
2345
2346             if self.dry_run:
2347                 print("Would create p4 label %s for tag" % name)
2348             elif self.prepare_p4_only:
2349                 print("Not creating p4 label %s for tag due to option" \
2350                       " --prepare-p4-only" % name)
2351             else:
2352                 p4_write_pipe(["label", "-i"], labelTemplate)
2353
2354                 # Use the label
2355                 p4_system(["tag", "-l", name] +
2356                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2357
2358                 if verbose:
2359                     print("created p4 label for tag %s" % name)
2360
2361     def run(self, args):
2362         if len(args) == 0:
2363             self.master = currentGitBranch()
2364         elif len(args) == 1:
2365             self.master = args[0]
2366             if not branchExists(self.master):
2367                 die("Branch %s does not exist" % self.master)
2368         else:
2369             return False
2370
2371         for i in self.update_shelve:
2372             if i <= 0:
2373                 sys.exit("invalid changelist %d" % i)
2374
2375         if self.master:
2376             allowSubmit = gitConfig("git-p4.allowSubmit")
2377             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2378                 die("%s is not in git-p4.allowSubmit" % self.master)
2379
2380         [upstream, settings] = findUpstreamBranchPoint()
2381         self.depotPath = settings['depot-paths'][0]
2382         if len(self.origin) == 0:
2383             self.origin = upstream
2384
2385         if len(self.update_shelve) > 0:
2386             self.shelve = True
2387
2388         if self.preserveUser:
2389             if not self.canChangeChangelists():
2390                 die("Cannot preserve user names without p4 super-user or admin permissions")
2391
2392         # if not set from the command line, try the config file
2393         if self.conflict_behavior is None:
2394             val = gitConfig("git-p4.conflict")
2395             if val:
2396                 if val not in self.conflict_behavior_choices:
2397                     die("Invalid value '%s' for config git-p4.conflict" % val)
2398             else:
2399                 val = "ask"
2400             self.conflict_behavior = val
2401
2402         if self.verbose:
2403             print("Origin branch is " + self.origin)
2404
2405         if len(self.depotPath) == 0:
2406             print("Internal error: cannot locate perforce depot path from existing branches")
2407             sys.exit(128)
2408
2409         self.useClientSpec = False
2410         if gitConfigBool("git-p4.useclientspec"):
2411             self.useClientSpec = True
2412         if self.useClientSpec:
2413             self.clientSpecDirs = getClientSpec()
2414
2415         # Check for the existence of P4 branches
2416         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2417
2418         if self.useClientSpec and not branchesDetected:
2419             # all files are relative to the client spec
2420             self.clientPath = getClientRoot()
2421         else:
2422             self.clientPath = p4Where(self.depotPath)
2423
2424         if self.clientPath == "":
2425             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2426
2427         print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2428         self.oldWorkingDirectory = os.getcwd()
2429
2430         # ensure the clientPath exists
2431         new_client_dir = False
2432         if not os.path.exists(self.clientPath):
2433             new_client_dir = True
2434             os.makedirs(self.clientPath)
2435
2436         chdir(self.clientPath, is_client_path=True)
2437         if self.dry_run:
2438             print("Would synchronize p4 checkout in %s" % self.clientPath)
2439         else:
2440             print("Synchronizing p4 checkout...")
2441             if new_client_dir:
2442                 # old one was destroyed, and maybe nobody told p4
2443                 p4_sync("...", "-f")
2444             else:
2445                 p4_sync("...")
2446         self.check()
2447
2448         commits = []
2449         if self.master:
2450             committish = self.master
2451         else:
2452             committish = 'HEAD'
2453
2454         if self.commit != "":
2455             if self.commit.find("..") != -1:
2456                 limits_ish = self.commit.split("..")
2457                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2458                     commits.append(line.strip())
2459                 commits.reverse()
2460             else:
2461                 commits.append(self.commit)
2462         else:
2463             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2464                 commits.append(line.strip())
2465             commits.reverse()
2466
2467         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2468             self.checkAuthorship = False
2469         else:
2470             self.checkAuthorship = True
2471
2472         if self.preserveUser:
2473             self.checkValidP4Users(commits)
2474
2475         #
2476         # Build up a set of options to be passed to diff when
2477         # submitting each commit to p4.
2478         #
2479         if self.detectRenames:
2480             # command-line -M arg
2481             self.diffOpts = "-M"
2482         else:
2483             # If not explicitly set check the config variable
2484             detectRenames = gitConfig("git-p4.detectRenames")
2485
2486             if detectRenames.lower() == "false" or detectRenames == "":
2487                 self.diffOpts = ""
2488             elif detectRenames.lower() == "true":
2489                 self.diffOpts = "-M"
2490             else:
2491                 self.diffOpts = "-M%s" % detectRenames
2492
2493         # no command-line arg for -C or --find-copies-harder, just
2494         # config variables
2495         detectCopies = gitConfig("git-p4.detectCopies")
2496         if detectCopies.lower() == "false" or detectCopies == "":
2497             pass
2498         elif detectCopies.lower() == "true":
2499             self.diffOpts += " -C"
2500         else:
2501             self.diffOpts += " -C%s" % detectCopies
2502
2503         if gitConfigBool("git-p4.detectCopiesHarder"):
2504             self.diffOpts += " --find-copies-harder"
2505
2506         num_shelves = len(self.update_shelve)
2507         if num_shelves > 0 and num_shelves != len(commits):
2508             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2509                      (len(commits), num_shelves))
2510
2511         if not self.no_verify:
2512             try:
2513                 if not run_git_hook("p4-pre-submit"):
2514                     print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \
2515                         "this pre-submission check by adding\nthe command line option '--no-verify', " \
2516                         "however,\nthis will also skip the p4-changelist hook as well.")
2517                     sys.exit(1)
2518             except Exception as e:
2519                 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\
2520                     "with the error '{0}'".format(e.message) )
2521                 sys.exit(1)
2522
2523         #
2524         # Apply the commits, one at a time.  On failure, ask if should
2525         # continue to try the rest of the patches, or quit.
2526         #
2527         if self.dry_run:
2528             print("Would apply")
2529         applied = []
2530         last = len(commits) - 1
2531         for i, commit in enumerate(commits):
2532             if self.dry_run:
2533                 print(" ", read_pipe(["git", "show", "-s",
2534                                       "--format=format:%h %s", commit]))
2535                 ok = True
2536             else:
2537                 ok = self.applyCommit(commit)
2538             if ok:
2539                 applied.append(commit)
2540                 if self.prepare_p4_only:
2541                     if i < last:
2542                         print("Processing only the first commit due to option" \
2543                                 " --prepare-p4-only")
2544                     break
2545             else:
2546                 if i < last:
2547                     # prompt for what to do, or use the option/variable
2548                     if self.conflict_behavior == "ask":
2549                         print("What do you want to do?")
2550                         response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2551                     elif self.conflict_behavior == "skip":
2552                         response = "s"
2553                     elif self.conflict_behavior == "quit":
2554                         response = "q"
2555                     else:
2556                         die("Unknown conflict_behavior '%s'" %
2557                             self.conflict_behavior)
2558
2559                     if response == "s":
2560                         print("Skipping this commit, but applying the rest")
2561                     if response == "q":
2562                         print("Quitting")
2563                         break
2564
2565         chdir(self.oldWorkingDirectory)
2566         shelved_applied = "shelved" if self.shelve else "applied"
2567         if self.dry_run:
2568             pass
2569         elif self.prepare_p4_only:
2570             pass
2571         elif len(commits) == len(applied):
2572             print("All commits {0}!".format(shelved_applied))
2573
2574             sync = P4Sync()
2575             if self.branch:
2576                 sync.branch = self.branch
2577             if self.disable_p4sync:
2578                 sync.sync_origin_only()
2579             else:
2580                 sync.run([])
2581
2582                 if not self.disable_rebase:
2583                     rebase = P4Rebase()
2584                     rebase.rebase()
2585
2586         else:
2587             if len(applied) == 0:
2588                 print("No commits {0}.".format(shelved_applied))
2589             else:
2590                 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2591                 for c in commits:
2592                     if c in applied:
2593                         star = "*"
2594                     else:
2595                         star = " "
2596                     print(star, read_pipe(["git", "show", "-s",
2597                                            "--format=format:%h %s",  c]))
2598                 print("You will have to do 'git p4 sync' and rebase.")
2599
2600         if gitConfigBool("git-p4.exportLabels"):
2601             self.exportLabels = True
2602
2603         if self.exportLabels:
2604             p4Labels = getP4Labels(self.depotPath)
2605             gitTags = getGitTags()
2606
2607             missingGitTags = gitTags - p4Labels
2608             self.exportGitTags(missingGitTags)
2609
2610         # exit with error unless everything applied perfectly
2611         if len(commits) != len(applied):
2612                 sys.exit(1)
2613
2614         return True
2615
2616 class View(object):
2617     """Represent a p4 view ("p4 help views"), and map files in a
2618        repo according to the view."""
2619
2620     def __init__(self, client_name):
2621         self.mappings = []
2622         self.client_prefix = "//%s/" % client_name
2623         # cache results of "p4 where" to lookup client file locations
2624         self.client_spec_path_cache = {}
2625
2626     def append(self, view_line):
2627         """Parse a view line, splitting it into depot and client
2628            sides.  Append to self.mappings, preserving order.  This
2629            is only needed for tag creation."""
2630
2631         # Split the view line into exactly two words.  P4 enforces
2632         # structure on these lines that simplifies this quite a bit.
2633         #
2634         # Either or both words may be double-quoted.
2635         # Single quotes do not matter.
2636         # Double-quote marks cannot occur inside the words.
2637         # A + or - prefix is also inside the quotes.
2638         # There are no quotes unless they contain a space.
2639         # The line is already white-space stripped.
2640         # The two words are separated by a single space.
2641         #
2642         if view_line[0] == '"':
2643             # First word is double quoted.  Find its end.
2644             close_quote_index = view_line.find('"', 1)
2645             if close_quote_index <= 0:
2646                 die("No first-word closing quote found: %s" % view_line)
2647             depot_side = view_line[1:close_quote_index]
2648             # skip closing quote and space
2649             rhs_index = close_quote_index + 1 + 1
2650         else:
2651             space_index = view_line.find(" ")
2652             if space_index <= 0:
2653                 die("No word-splitting space found: %s" % view_line)
2654             depot_side = view_line[0:space_index]
2655             rhs_index = space_index + 1
2656
2657         # prefix + means overlay on previous mapping
2658         if depot_side.startswith("+"):
2659             depot_side = depot_side[1:]
2660
2661         # prefix - means exclude this path, leave out of mappings
2662         exclude = False
2663         if depot_side.startswith("-"):
2664             exclude = True
2665             depot_side = depot_side[1:]
2666
2667         if not exclude:
2668             self.mappings.append(depot_side)
2669
2670     def convert_client_path(self, clientFile):
2671         # chop off //client/ part to make it relative
2672         if not decode_path(clientFile).startswith(self.client_prefix):
2673             die("No prefix '%s' on clientFile '%s'" %
2674                 (self.client_prefix, clientFile))
2675         return clientFile[len(self.client_prefix):]
2676
2677     def update_client_spec_path_cache(self, files):
2678         """ Caching file paths by "p4 where" batch query """
2679
2680         # List depot file paths exclude that already cached
2681         fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
2682
2683         if len(fileArgs) == 0:
2684             return  # All files in cache
2685
2686         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2687         for res in where_result:
2688             if "code" in res and res["code"] == "error":
2689                 # assume error is "... file(s) not in client view"
2690                 continue
2691             if "clientFile" not in res:
2692                 die("No clientFile in 'p4 where' output")
2693             if "unmap" in res:
2694                 # it will list all of them, but only one not unmap-ped
2695                 continue
2696             depot_path = decode_path(res['depotFile'])
2697             if gitConfigBool("core.ignorecase"):
2698                 depot_path = depot_path.lower()
2699             self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
2700
2701         # not found files or unmap files set to ""
2702         for depotFile in fileArgs:
2703             depotFile = decode_path(depotFile)
2704             if gitConfigBool("core.ignorecase"):
2705                 depotFile = depotFile.lower()
2706             if depotFile not in self.client_spec_path_cache:
2707                 self.client_spec_path_cache[depotFile] = b''
2708
2709     def map_in_client(self, depot_path):
2710         """Return the relative location in the client where this
2711            depot file should live.  Returns "" if the file should
2712            not be mapped in the client."""
2713
2714         if gitConfigBool("core.ignorecase"):
2715             depot_path = depot_path.lower()
2716
2717         if depot_path in self.client_spec_path_cache:
2718             return self.client_spec_path_cache[depot_path]
2719
2720         die( "Error: %s is not found in client spec path" % depot_path )
2721         return ""
2722
2723 def cloneExcludeCallback(option, opt_str, value, parser):
2724     # prepend "/" because the first "/" was consumed as part of the option itself.
2725     # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2726     parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2727
2728 class P4Sync(Command, P4UserMap):
2729
2730     def __init__(self):
2731         Command.__init__(self)
2732         P4UserMap.__init__(self)
2733         self.options = [
2734                 optparse.make_option("--branch", dest="branch"),
2735                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2736                 optparse.make_option("--changesfile", dest="changesFile"),
2737                 optparse.make_option("--silent", dest="silent", action="store_true"),
2738                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2739                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2740                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2741                                      help="Import into refs/heads/ , not refs/remotes"),
2742                 optparse.make_option("--max-changes", dest="maxChanges",
2743                                      help="Maximum number of changes to import"),
2744                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2745                                      help="Internal block size to use when iteratively calling p4 changes"),
2746                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2747                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2748                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2749                                      help="Only sync files that are included in the Perforce Client Spec"),
2750                 optparse.make_option("-/", dest="cloneExclude",
2751                                      action="callback", callback=cloneExcludeCallback, type="string",
2752                                      help="exclude depot path"),
2753         ]
2754         self.description = """Imports from Perforce into a git repository.\n
2755     example:
2756     //depot/my/project/ -- to import the current head
2757     //depot/my/project/@all -- to import everything
2758     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2759
2760     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2761
2762         self.usage += " //depot/path[@revRange]"
2763         self.silent = False
2764         self.createdBranches = set()
2765         self.committedChanges = set()
2766         self.branch = ""
2767         self.detectBranches = False
2768         self.detectLabels = False
2769         self.importLabels = False
2770         self.changesFile = ""
2771         self.syncWithOrigin = True
2772         self.importIntoRemotes = True
2773         self.maxChanges = ""
2774         self.changes_block_size = None
2775         self.keepRepoPath = False
2776         self.depotPaths = None
2777         self.p4BranchesInGit = []
2778         self.cloneExclude = []
2779         self.useClientSpec = False
2780         self.useClientSpec_from_options = False
2781         self.clientSpecDirs = None
2782         self.tempBranches = []
2783         self.tempBranchLocation = "refs/git-p4-tmp"
2784         self.largeFileSystem = None
2785         self.suppress_meta_comment = False
2786
2787         if gitConfig('git-p4.largeFileSystem'):
2788             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2789             self.largeFileSystem = largeFileSystemConstructor(
2790                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2791             )
2792
2793         if gitConfig("git-p4.syncFromOrigin") == "false":
2794             self.syncWithOrigin = False
2795
2796         self.depotPaths = []
2797         self.changeRange = ""
2798         self.previousDepotPaths = []
2799         self.hasOrigin = False
2800
2801         # map from branch depot path to parent branch
2802         self.knownBranches = {}
2803         self.initialParents = {}
2804
2805         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2806         self.labels = {}
2807
2808     # Force a checkpoint in fast-import and wait for it to finish
2809     def checkpoint(self):
2810         self.gitStream.write("checkpoint\n\n")
2811         self.gitStream.write("progress checkpoint\n\n")
2812         self.gitStream.flush()
2813         out = self.gitOutput.readline()
2814         if self.verbose:
2815             print("checkpoint finished: " + out)
2816
2817     def isPathWanted(self, path):
2818         for p in self.cloneExclude:
2819             if p.endswith("/"):
2820                 if p4PathStartsWith(path, p):
2821                     return False
2822             # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2823             elif path.lower() == p.lower():
2824                 return False
2825         for p in self.depotPaths:
2826             if p4PathStartsWith(path, decode_path(p)):
2827                 return True
2828         return False
2829
2830     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2831         files = []
2832         fnum = 0
2833         while "depotFile%s" % fnum in commit:
2834             path =  commit["depotFile%s" % fnum]
2835             found = self.isPathWanted(decode_path(path))
2836             if not found:
2837                 fnum = fnum + 1
2838                 continue
2839
2840             file = {}
2841             file["path"] = path
2842             file["rev"] = commit["rev%s" % fnum]
2843             file["action"] = commit["action%s" % fnum]
2844             file["type"] = commit["type%s" % fnum]
2845             if shelved:
2846                 file["shelved_cl"] = int(shelved_cl)
2847             files.append(file)
2848             fnum = fnum + 1
2849         return files
2850
2851     def extractJobsFromCommit(self, commit):
2852         jobs = []
2853         jnum = 0
2854         while "job%s" % jnum in commit:
2855             job = commit["job%s" % jnum]
2856             jobs.append(job)
2857             jnum = jnum + 1
2858         return jobs
2859
2860     def stripRepoPath(self, path, prefixes):
2861         """When streaming files, this is called to map a p4 depot path
2862            to where it should go in git.  The prefixes are either
2863            self.depotPaths, or self.branchPrefixes in the case of
2864            branch detection."""
2865
2866         if self.useClientSpec:
2867             # branch detection moves files up a level (the branch name)
2868             # from what client spec interpretation gives
2869             path = decode_path(self.clientSpecDirs.map_in_client(path))
2870             if self.detectBranches:
2871                 for b in self.knownBranches:
2872                     if p4PathStartsWith(path, b + "/"):
2873                         path = path[len(b)+1:]
2874
2875         elif self.keepRepoPath:
2876             # Preserve everything in relative path name except leading
2877             # //depot/; just look at first prefix as they all should
2878             # be in the same depot.
2879             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2880             if p4PathStartsWith(path, depot):
2881                 path = path[len(depot):]
2882
2883         else:
2884             for p in prefixes:
2885                 if p4PathStartsWith(path, p):
2886                     path = path[len(p):]
2887                     break
2888
2889         path = wildcard_decode(path)
2890         return path
2891
2892     def splitFilesIntoBranches(self, commit):
2893         """Look at each depotFile in the commit to figure out to what
2894            branch it belongs."""
2895
2896         if self.clientSpecDirs:
2897             files = self.extractFilesFromCommit(commit)
2898             self.clientSpecDirs.update_client_spec_path_cache(files)
2899
2900         branches = {}
2901         fnum = 0
2902         while "depotFile%s" % fnum in commit:
2903             raw_path = commit["depotFile%s" % fnum]
2904             path = decode_path(raw_path)
2905             found = self.isPathWanted(path)
2906             if not found:
2907                 fnum = fnum + 1
2908                 continue
2909
2910             file = {}
2911             file["path"] = raw_path
2912             file["rev"] = commit["rev%s" % fnum]
2913             file["action"] = commit["action%s" % fnum]
2914             file["type"] = commit["type%s" % fnum]
2915             fnum = fnum + 1
2916
2917             # start with the full relative path where this file would
2918             # go in a p4 client
2919             if self.useClientSpec:
2920                 relPath = decode_path(self.clientSpecDirs.map_in_client(path))
2921             else:
2922                 relPath = self.stripRepoPath(path, self.depotPaths)
2923
2924             for branch in self.knownBranches.keys():
2925                 # add a trailing slash so that a commit into qt/4.2foo
2926                 # doesn't end up in qt/4.2, e.g.
2927                 if p4PathStartsWith(relPath, branch + "/"):
2928                     if branch not in branches:
2929                         branches[branch] = []
2930                     branches[branch].append(file)
2931                     break
2932
2933         return branches
2934
2935     def writeToGitStream(self, gitMode, relPath, contents):
2936         self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
2937         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2938         for d in contents:
2939             self.gitStream.write(d)
2940         self.gitStream.write('\n')
2941
2942     def encodeWithUTF8(self, path):
2943         try:
2944             path.decode('ascii')
2945         except:
2946             encoding = 'utf8'
2947             if gitConfig('git-p4.pathEncoding'):
2948                 encoding = gitConfig('git-p4.pathEncoding')
2949             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2950             if self.verbose:
2951                 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2952         return path
2953
2954     # output one file from the P4 stream
2955     # - helper for streamP4Files
2956
2957     def streamOneP4File(self, file, contents):
2958         file_path = file['depotFile']
2959         relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
2960
2961         if verbose:
2962             if 'fileSize' in self.stream_file:
2963                 size = int(self.stream_file['fileSize'])
2964             else:
2965                 size = 0 # deleted files don't get a fileSize apparently
2966             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file_path, relPath, size/1024/1024))
2967             sys.stdout.flush()
2968
2969         (type_base, type_mods) = split_p4_type(file["type"])
2970
2971         git_mode = "100644"
2972         if "x" in type_mods:
2973             git_mode = "100755"
2974         if type_base == "symlink":
2975             git_mode = "120000"
2976             # p4 print on a symlink sometimes contains "target\n";
2977             # if it does, remove the newline
2978             data = ''.join(decode_text_stream(c) for c in contents)
2979             if not data:
2980                 # Some version of p4 allowed creating a symlink that pointed
2981                 # to nothing.  This causes p4 errors when checking out such
2982                 # a change, and errors here too.  Work around it by ignoring
2983                 # the bad symlink; hopefully a future change fixes it.
2984                 print("\nIgnoring empty symlink in %s" % file_path)
2985                 return
2986             elif data[-1] == '\n':
2987                 contents = [data[:-1]]
2988             else:
2989                 contents = [data]
2990
2991         if type_base == "utf16":
2992             # p4 delivers different text in the python output to -G
2993             # than it does when using "print -o", or normal p4 client
2994             # operations.  utf16 is converted to ascii or utf8, perhaps.
2995             # But ascii text saved as -t utf16 is completely mangled.
2996             # Invoke print -o to get the real contents.
2997             #
2998             # On windows, the newlines will always be mangled by print, so put
2999             # them back too.  This is not needed to the cygwin windows version,
3000             # just the native "NT" type.
3001             #
3002             try:
3003                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
3004             except Exception as e:
3005                 if 'Translation of file content failed' in str(e):
3006                     type_base = 'binary'
3007                 else:
3008                     raise e
3009             else:
3010                 if p4_version_string().find('/NT') >= 0:
3011                     text = text.replace(b'\r\n', b'\n')
3012                 contents = [ text ]
3013
3014         if type_base == "apple":
3015             # Apple filetype files will be streamed as a concatenation of
3016             # its appledouble header and the contents.  This is useless
3017             # on both macs and non-macs.  If using "print -q -o xx", it
3018             # will create "xx" with the data, and "%xx" with the header.
3019             # This is also not very useful.
3020             #
3021             # Ideally, someday, this script can learn how to generate
3022             # appledouble files directly and import those to git, but
3023             # non-mac machines can never find a use for apple filetype.
3024             print("\nIgnoring apple filetype file %s" % file['depotFile'])
3025             return
3026
3027         # Note that we do not try to de-mangle keywords on utf16 files,
3028         # even though in theory somebody may want that.
3029         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
3030         if pattern:
3031             regexp = re.compile(pattern, re.VERBOSE)
3032             text = ''.join(decode_text_stream(c) for c in contents)
3033             text = regexp.sub(r'$\1$', text)
3034             contents = [ encode_text_stream(text) ]
3035
3036         if self.largeFileSystem:
3037             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
3038
3039         self.writeToGitStream(git_mode, relPath, contents)
3040
3041     def streamOneP4Deletion(self, file):
3042         relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
3043         if verbose:
3044             sys.stdout.write("delete %s\n" % relPath)
3045             sys.stdout.flush()
3046         self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
3047
3048         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
3049             self.largeFileSystem.removeLargeFile(relPath)
3050
3051     # handle another chunk of streaming data
3052     def streamP4FilesCb(self, marshalled):
3053
3054         # catch p4 errors and complain
3055         err = None
3056         if "code" in marshalled:
3057             if marshalled["code"] == "error":
3058                 if "data" in marshalled:
3059                     err = marshalled["data"].rstrip()
3060
3061         if not err and 'fileSize' in self.stream_file:
3062             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
3063             if required_bytes > 0:
3064                 err = 'Not enough space left on %s! Free at least %i MB.' % (
3065                     os.getcwd(), required_bytes/1024/1024
3066                 )
3067
3068         if err:
3069             f = None
3070             if self.stream_have_file_info:
3071                 if "depotFile" in self.stream_file:
3072                     f = self.stream_file["depotFile"]
3073             # force a failure in fast-import, else an empty
3074             # commit will be made
3075             self.gitStream.write("\n")
3076             self.gitStream.write("die-now\n")
3077             self.gitStream.close()
3078             # ignore errors, but make sure it exits first
3079             self.importProcess.wait()
3080             if f:
3081                 die("Error from p4 print for %s: %s" % (f, err))
3082             else:
3083                 die("Error from p4 print: %s" % err)
3084
3085         if 'depotFile' in marshalled and self.stream_have_file_info:
3086             # start of a new file - output the old one first
3087             self.streamOneP4File(self.stream_file, self.stream_contents)
3088             self.stream_file = {}
3089             self.stream_contents = []
3090             self.stream_have_file_info = False
3091
3092         # pick up the new file information... for the
3093         # 'data' field we need to append to our array
3094         for k in marshalled.keys():
3095             if k == 'data':
3096                 if 'streamContentSize' not in self.stream_file:
3097                     self.stream_file['streamContentSize'] = 0
3098                 self.stream_file['streamContentSize'] += len(marshalled['data'])
3099                 self.stream_contents.append(marshalled['data'])
3100             else:
3101                 self.stream_file[k] = marshalled[k]
3102
3103         if (verbose and
3104             'streamContentSize' in self.stream_file and
3105             'fileSize' in self.stream_file and
3106             'depotFile' in self.stream_file):
3107             size = int(self.stream_file["fileSize"])
3108             if size > 0:
3109                 progress = 100*self.stream_file['streamContentSize']/size
3110                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
3111                 sys.stdout.flush()
3112
3113         self.stream_have_file_info = True
3114
3115     # Stream directly from "p4 files" into "git fast-import"
3116     def streamP4Files(self, files):
3117         filesForCommit = []
3118         filesToRead = []
3119         filesToDelete = []
3120
3121         for f in files:
3122             filesForCommit.append(f)
3123             if f['action'] in self.delete_actions:
3124                 filesToDelete.append(f)
3125             else:
3126                 filesToRead.append(f)
3127
3128         # deleted files...
3129         for f in filesToDelete:
3130             self.streamOneP4Deletion(f)
3131
3132         if len(filesToRead) > 0:
3133             self.stream_file = {}
3134             self.stream_contents = []
3135             self.stream_have_file_info = False
3136
3137             # curry self argument
3138             def streamP4FilesCbSelf(entry):
3139                 self.streamP4FilesCb(entry)
3140
3141             fileArgs = []
3142             for f in filesToRead:
3143                 if 'shelved_cl' in f:
3144                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
3145                     # the contents
3146                     fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
3147                 else:
3148                     fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
3149
3150                 fileArgs.append(fileArg)
3151
3152             p4CmdList(["-x", "-", "print"],
3153                       stdin=fileArgs,
3154                       cb=streamP4FilesCbSelf)
3155
3156             # do the last chunk
3157             if 'depotFile' in self.stream_file:
3158                 self.streamOneP4File(self.stream_file, self.stream_contents)
3159
3160     def make_email(self, userid):
3161         if userid in self.users:
3162             return self.users[userid]
3163         else:
3164             return "%s <a@b>" % userid
3165
3166     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
3167         """ Stream a p4 tag.
3168         commit is either a git commit, or a fast-import mark, ":<p4commit>"
3169         """
3170
3171         if verbose:
3172             print("writing tag %s for commit %s" % (labelName, commit))
3173         gitStream.write("tag %s\n" % labelName)
3174         gitStream.write("from %s\n" % commit)
3175
3176         if 'Owner' in labelDetails:
3177             owner = labelDetails["Owner"]
3178         else:
3179             owner = None
3180
3181         # Try to use the owner of the p4 label, or failing that,
3182         # the current p4 user id.
3183         if owner:
3184             email = self.make_email(owner)
3185         else:
3186             email = self.make_email(self.p4UserId())
3187         tagger = "%s %s %s" % (email, epoch, self.tz)
3188
3189         gitStream.write("tagger %s\n" % tagger)
3190
3191         print("labelDetails=",labelDetails)
3192         if 'Description' in labelDetails:
3193             description = labelDetails['Description']
3194         else:
3195             description = 'Label from git p4'
3196
3197         gitStream.write("data %d\n" % len(description))
3198         gitStream.write(description)
3199         gitStream.write("\n")
3200
3201     def inClientSpec(self, path):
3202         if not self.clientSpecDirs:
3203             return True
3204         inClientSpec = self.clientSpecDirs.map_in_client(path)
3205         if not inClientSpec and self.verbose:
3206             print('Ignoring file outside of client spec: {0}'.format(path))
3207         return inClientSpec
3208
3209     def hasBranchPrefix(self, path):
3210         if not self.branchPrefixes:
3211             return True
3212         hasPrefix = [p for p in self.branchPrefixes
3213                         if p4PathStartsWith(path, p)]
3214         if not hasPrefix and self.verbose:
3215             print('Ignoring file outside of prefix: {0}'.format(path))
3216         return hasPrefix
3217
3218     def findShadowedFiles(self, files, change):
3219         # Perforce allows you commit files and directories with the same name,
3220         # so you could have files //depot/foo and //depot/foo/bar both checked
3221         # in.  A p4 sync of a repository in this state fails.  Deleting one of
3222         # the files recovers the repository.
3223         #
3224         # Git will not allow the broken state to exist and only the most recent
3225         # of the conflicting names is left in the repository.  When one of the
3226         # conflicting files is deleted we need to re-add the other one to make
3227         # sure the git repository recovers in the same way as perforce.
3228         deleted = [f for f in files if f['action'] in self.delete_actions]
3229         to_check = set()
3230         for f in deleted:
3231             path = decode_path(f['path'])
3232             to_check.add(path + '/...')
3233             while True:
3234                 path = path.rsplit("/", 1)[0]
3235                 if path == "/" or path in to_check:
3236                     break
3237                 to_check.add(path)
3238         to_check = ['%s@%s' % (wildcard_encode(p), change) for p in to_check
3239             if self.hasBranchPrefix(p)]
3240         if to_check:
3241             stat_result = p4CmdList(["-x", "-", "fstat", "-T",
3242                 "depotFile,headAction,headRev,headType"], stdin=to_check)
3243             for record in stat_result:
3244                 if record['code'] != 'stat':
3245                     continue
3246                 if record['headAction'] in self.delete_actions:
3247                     continue
3248                 files.append({
3249                     'action': 'add',
3250                     'path': record['depotFile'],
3251                     'rev': record['headRev'],
3252                     'type': record['headType']})
3253
3254     def commit(self, details, files, branch, parent = "", allow_empty=False):
3255         epoch = details["time"]
3256         author = details["user"]
3257         jobs = self.extractJobsFromCommit(details)
3258
3259         if self.verbose:
3260             print('commit into {0}'.format(branch))
3261
3262         files = [f for f in files
3263             if self.hasBranchPrefix(decode_path(f['path']))]
3264         self.findShadowedFiles(files, details['change'])
3265
3266         if self.clientSpecDirs:
3267             self.clientSpecDirs.update_client_spec_path_cache(files)
3268
3269         files = [f for f in files if self.inClientSpec(decode_path(f['path']))]
3270
3271         if gitConfigBool('git-p4.keepEmptyCommits'):
3272             allow_empty = True
3273
3274         if not files and not allow_empty:
3275             print('Ignoring revision {0} as it would produce an empty commit.'
3276                 .format(details['change']))
3277             return
3278
3279         self.gitStream.write("commit %s\n" % branch)
3280         self.gitStream.write("mark :%s\n" % details["change"])
3281         self.committedChanges.add(int(details["change"]))
3282         committer = ""
3283         if author not in self.users:
3284             self.getUserMapFromPerforceServer()
3285         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3286
3287         self.gitStream.write("committer %s\n" % committer)
3288
3289         self.gitStream.write("data <<EOT\n")
3290         self.gitStream.write(details["desc"])
3291         if len(jobs) > 0:
3292             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3293
3294         if not self.suppress_meta_comment:
3295             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3296                                 (','.join(self.branchPrefixes), details["change"]))
3297             if len(details['options']) > 0:
3298                 self.gitStream.write(": options = %s" % details['options'])
3299             self.gitStream.write("]\n")
3300
3301         self.gitStream.write("EOT\n\n")
3302
3303         if len(parent) > 0:
3304             if self.verbose:
3305                 print("parent %s" % parent)
3306             self.gitStream.write("from %s\n" % parent)
3307
3308         self.streamP4Files(files)
3309         self.gitStream.write("\n")
3310
3311         change = int(details["change"])
3312
3313         if change in self.labels:
3314             label = self.labels[change]
3315             labelDetails = label[0]
3316             labelRevisions = label[1]
3317             if self.verbose:
3318                 print("Change %s is labelled %s" % (change, labelDetails))
3319
3320             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3321                                                 for p in self.branchPrefixes])
3322
3323             if len(files) == len(labelRevisions):
3324
3325                 cleanedFiles = {}
3326                 for info in files:
3327                     if info["action"] in self.delete_actions:
3328                         continue
3329                     cleanedFiles[info["depotFile"]] = info["rev"]
3330
3331                 if cleanedFiles == labelRevisions:
3332                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3333
3334                 else:
3335                     if not self.silent:
3336                         print("Tag %s does not match with change %s: files do not match."
3337                                % (labelDetails["label"], change))
3338
3339             else:
3340                 if not self.silent:
3341                     print("Tag %s does not match with change %s: file count is different."
3342                            % (labelDetails["label"], change))
3343
3344     # Build a dictionary of changelists and labels, for "detect-labels" option.
3345     def getLabels(self):
3346         self.labels = {}
3347
3348         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3349         if len(l) > 0 and not self.silent:
3350             print("Finding files belonging to labels in %s" % self.depotPaths)
3351
3352         for output in l:
3353             label = output["label"]
3354             revisions = {}
3355             newestChange = 0
3356             if self.verbose:
3357                 print("Querying files for label %s" % label)
3358             for file in p4CmdList(["files"] +
3359                                       ["%s...@%s" % (p, label)
3360                                           for p in self.depotPaths]):
3361                 revisions[file["depotFile"]] = file["rev"]
3362                 change = int(file["change"])
3363                 if change > newestChange:
3364                     newestChange = change
3365
3366             self.labels[newestChange] = [output, revisions]
3367
3368         if self.verbose:
3369             print("Label changes: %s" % self.labels.keys())
3370
3371     # Import p4 labels as git tags. A direct mapping does not
3372     # exist, so assume that if all the files are at the same revision
3373     # then we can use that, or it's something more complicated we should
3374     # just ignore.
3375     def importP4Labels(self, stream, p4Labels):
3376         if verbose:
3377             print("import p4 labels: " + ' '.join(p4Labels))
3378
3379         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3380         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3381         if len(validLabelRegexp) == 0:
3382             validLabelRegexp = defaultLabelRegexp
3383         m = re.compile(validLabelRegexp)
3384
3385         for name in p4Labels:
3386             commitFound = False
3387
3388             if not m.match(name):
3389                 if verbose:
3390                     print("label %s does not match regexp %s" % (name,validLabelRegexp))
3391                 continue
3392
3393             if name in ignoredP4Labels:
3394                 continue
3395
3396             labelDetails = p4CmdList(['label', "-o", name])[0]
3397
3398             # get the most recent changelist for each file in this label
3399             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3400                                 for p in self.depotPaths])
3401
3402             if 'change' in change:
3403                 # find the corresponding git commit; take the oldest commit
3404                 changelist = int(change['change'])
3405                 if changelist in self.committedChanges:
3406                     gitCommit = ":%d" % changelist       # use a fast-import mark
3407                     commitFound = True
3408                 else:
3409                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3410                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3411                     if len(gitCommit) == 0:
3412                         print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3413                     else:
3414                         commitFound = True
3415                         gitCommit = gitCommit.strip()
3416
3417                 if commitFound:
3418                     # Convert from p4 time format
3419                     try:
3420                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3421                     except ValueError:
3422                         print("Could not convert label time %s" % labelDetails['Update'])
3423                         tmwhen = 1
3424
3425                     when = int(time.mktime(tmwhen))
3426                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3427                     if verbose:
3428                         print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3429             else:
3430                 if verbose:
3431                     print("Label %s has no changelists - possibly deleted?" % name)
3432
3433             if not commitFound:
3434                 # We can't import this label; don't try again as it will get very
3435                 # expensive repeatedly fetching all the files for labels that will
3436                 # never be imported. If the label is moved in the future, the
3437                 # ignore will need to be removed manually.
3438                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3439
3440     def guessProjectName(self):
3441         for p in self.depotPaths:
3442             if p.endswith("/"):
3443                 p = p[:-1]
3444             p = p[p.strip().rfind("/") + 1:]
3445             if not p.endswith("/"):
3446                p += "/"
3447             return p
3448
3449     def getBranchMapping(self):
3450         lostAndFoundBranches = set()
3451
3452         user = gitConfig("git-p4.branchUser")
3453         if len(user) > 0:
3454             command = "branches -u %s" % user
3455         else:
3456             command = "branches"
3457
3458         for info in p4CmdList(command):
3459             details = p4Cmd(["branch", "-o", info["branch"]])
3460             viewIdx = 0
3461             while "View%s" % viewIdx in details:
3462                 paths = details["View%s" % viewIdx].split(" ")
3463                 viewIdx = viewIdx + 1
3464                 # require standard //depot/foo/... //depot/bar/... mapping
3465                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3466                     continue
3467                 source = paths[0]
3468                 destination = paths[1]
3469                 ## HACK
3470                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3471                     source = source[len(self.depotPaths[0]):-4]
3472                     destination = destination[len(self.depotPaths[0]):-4]
3473
3474                     if destination in self.knownBranches:
3475                         if not self.silent:
3476                             print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3477                             print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3478                         continue
3479
3480                     self.knownBranches[destination] = source
3481
3482                     lostAndFoundBranches.discard(destination)
3483
3484                     if source not in self.knownBranches:
3485                         lostAndFoundBranches.add(source)
3486
3487         # Perforce does not strictly require branches to be defined, so we also
3488         # check git config for a branch list.
3489         #
3490         # Example of branch definition in git config file:
3491         # [git-p4]
3492         #   branchList=main:branchA
3493         #   branchList=main:branchB
3494         #   branchList=branchA:branchC
3495         configBranches = gitConfigList("git-p4.branchList")
3496         for branch in configBranches:
3497             if branch:
3498                 (source, destination) = branch.split(":")
3499                 self.knownBranches[destination] = source
3500
3501                 lostAndFoundBranches.discard(destination)
3502
3503                 if source not in self.knownBranches:
3504                     lostAndFoundBranches.add(source)
3505
3506
3507         for branch in lostAndFoundBranches:
3508             self.knownBranches[branch] = branch
3509
3510     def getBranchMappingFromGitBranches(self):
3511         branches = p4BranchesInGit(self.importIntoRemotes)
3512         for branch in branches.keys():
3513             if branch == "master":
3514                 branch = "main"
3515             else:
3516                 branch = branch[len(self.projectName):]
3517             self.knownBranches[branch] = branch
3518
3519     def updateOptionDict(self, d):
3520         option_keys = {}
3521         if self.keepRepoPath:
3522             option_keys['keepRepoPath'] = 1
3523
3524         d["options"] = ' '.join(sorted(option_keys.keys()))
3525
3526     def readOptions(self, d):
3527         self.keepRepoPath = ('options' in d
3528                              and ('keepRepoPath' in d['options']))
3529
3530     def gitRefForBranch(self, branch):
3531         if branch == "main":
3532             return self.refPrefix + "master"
3533
3534         if len(branch) <= 0:
3535             return branch
3536
3537         return self.refPrefix + self.projectName + branch
3538
3539     def gitCommitByP4Change(self, ref, change):
3540         if self.verbose:
3541             print("looking in ref " + ref + " for change %s using bisect..." % change)
3542
3543         earliestCommit = ""
3544         latestCommit = parseRevision(ref)
3545
3546         while True:
3547             if self.verbose:
3548                 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3549             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3550             if len(next) == 0:
3551                 if self.verbose:
3552                     print("argh")
3553                 return ""
3554             log = extractLogMessageFromGitCommit(next)
3555             settings = extractSettingsGitLog(log)
3556             currentChange = int(settings['change'])
3557             if self.verbose:
3558                 print("current change %s" % currentChange)
3559
3560             if currentChange == change:
3561                 if self.verbose:
3562                     print("found %s" % next)
3563                 return next
3564
3565             if currentChange < change:
3566                 earliestCommit = "^%s" % next
3567             else:
3568                 if next == latestCommit:
3569                     die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3570                 latestCommit = "%s^@" % next
3571
3572         return ""
3573
3574     def importNewBranch(self, branch, maxChange):
3575         # make fast-import flush all changes to disk and update the refs using the checkpoint
3576         # command so that we can try to find the branch parent in the git history
3577         self.gitStream.write("checkpoint\n\n");
3578         self.gitStream.flush();
3579         branchPrefix = self.depotPaths[0] + branch + "/"
3580         range = "@1,%s" % maxChange
3581         #print "prefix" + branchPrefix
3582         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3583         if len(changes) <= 0:
3584             return False
3585         firstChange = changes[0]
3586         #print "first change in branch: %s" % firstChange
3587         sourceBranch = self.knownBranches[branch]
3588         sourceDepotPath = self.depotPaths[0] + sourceBranch
3589         sourceRef = self.gitRefForBranch(sourceBranch)
3590         #print "source " + sourceBranch
3591
3592         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3593         #print "branch parent: %s" % branchParentChange
3594         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3595         if len(gitParent) > 0:
3596             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3597             #print "parent git commit: %s" % gitParent
3598
3599         self.importChanges(changes)
3600         return True
3601
3602     def searchParent(self, parent, branch, target):
3603         parentFound = False
3604         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3605                                      "--no-merges", parent]):
3606             blob = blob.strip()
3607             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3608                 parentFound = True
3609                 if self.verbose:
3610                     print("Found parent of %s in commit %s" % (branch, blob))
3611                 break
3612         if parentFound:
3613             return blob
3614         else:
3615             return None
3616
3617     def importChanges(self, changes, origin_revision=0):
3618         cnt = 1
3619         for change in changes:
3620             description = p4_describe(change)
3621             self.updateOptionDict(description)
3622
3623             if not self.silent:
3624                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3625                 sys.stdout.flush()
3626             cnt = cnt + 1
3627
3628             try:
3629                 if self.detectBranches:
3630                     branches = self.splitFilesIntoBranches(description)
3631                     for branch in branches.keys():
3632                         ## HACK  --hwn
3633                         branchPrefix = self.depotPaths[0] + branch + "/"
3634                         self.branchPrefixes = [ branchPrefix ]
3635
3636                         parent = ""
3637
3638                         filesForCommit = branches[branch]
3639
3640                         if self.verbose:
3641                             print("branch is %s" % branch)
3642
3643                         self.updatedBranches.add(branch)
3644
3645                         if branch not in self.createdBranches:
3646                             self.createdBranches.add(branch)
3647                             parent = self.knownBranches[branch]
3648                             if parent == branch:
3649                                 parent = ""
3650                             else:
3651                                 fullBranch = self.projectName + branch
3652                                 if fullBranch not in self.p4BranchesInGit:
3653                                     if not self.silent:
3654                                         print("\n    Importing new branch %s" % fullBranch);
3655                                     if self.importNewBranch(branch, change - 1):
3656                                         parent = ""
3657                                         self.p4BranchesInGit.append(fullBranch)
3658                                     if not self.silent:
3659                                         print("\n    Resuming with change %s" % change);
3660
3661                                 if self.verbose:
3662                                     print("parent determined through known branches: %s" % parent)
3663
3664                         branch = self.gitRefForBranch(branch)
3665                         parent = self.gitRefForBranch(parent)
3666
3667                         if self.verbose:
3668                             print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3669
3670                         if len(parent) == 0 and branch in self.initialParents:
3671                             parent = self.initialParents[branch]
3672                             del self.initialParents[branch]
3673
3674                         blob = None
3675                         if len(parent) > 0:
3676                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3677                             if self.verbose:
3678                                 print("Creating temporary branch: " + tempBranch)
3679                             self.commit(description, filesForCommit, tempBranch)
3680                             self.tempBranches.append(tempBranch)
3681                             self.checkpoint()
3682                             blob = self.searchParent(parent, branch, tempBranch)
3683                         if blob:
3684                             self.commit(description, filesForCommit, branch, blob)
3685                         else:
3686                             if self.verbose:
3687                                 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3688                             self.commit(description, filesForCommit, branch, parent)
3689                 else:
3690                     files = self.extractFilesFromCommit(description)
3691                     self.commit(description, files, self.branch,
3692                                 self.initialParent)
3693                     # only needed once, to connect to the previous commit
3694                     self.initialParent = ""
3695             except IOError:
3696                 print(self.gitError.read())
3697                 sys.exit(1)
3698
3699     def sync_origin_only(self):
3700         if self.syncWithOrigin:
3701             self.hasOrigin = originP4BranchesExist()
3702             if self.hasOrigin:
3703                 if not self.silent:
3704                     print('Syncing with origin first, using "git fetch origin"')
3705                 system("git fetch origin")
3706
3707     def importHeadRevision(self, revision):
3708         print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3709
3710         details = {}
3711         details["user"] = "git perforce import user"
3712         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3713                            % (' '.join(self.depotPaths), revision))
3714         details["change"] = revision
3715         newestRevision = 0
3716
3717         fileCnt = 0
3718         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3719
3720         for info in p4CmdList(["files"] + fileArgs):
3721
3722             if 'code' in info and info['code'] == 'error':
3723                 sys.stderr.write("p4 returned an error: %s\n"
3724                                  % info['data'])
3725                 if info['data'].find("must refer to client") >= 0:
3726                     sys.stderr.write("This particular p4 error is misleading.\n")
3727                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3728                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3729                 sys.exit(1)
3730             if 'p4ExitCode' in info:
3731                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3732                 sys.exit(1)
3733
3734
3735             change = int(info["change"])
3736             if change > newestRevision:
3737                 newestRevision = change
3738
3739             if info["action"] in self.delete_actions:
3740                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3741                 #fileCnt = fileCnt + 1
3742                 continue
3743
3744             for prop in ["depotFile", "rev", "action", "type" ]:
3745                 details["%s%s" % (prop, fileCnt)] = info[prop]
3746
3747             fileCnt = fileCnt + 1
3748
3749         details["change"] = newestRevision
3750
3751         # Use time from top-most change so that all git p4 clones of
3752         # the same p4 repo have the same commit SHA1s.
3753         res = p4_describe(newestRevision)
3754         details["time"] = res["time"]
3755
3756         self.updateOptionDict(details)
3757         try:
3758             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3759         except IOError as err:
3760             print("IO error with git fast-import. Is your git version recent enough?")
3761             print("IO error details: {}".format(err))
3762             print(self.gitError.read())
3763
3764
3765     def importRevisions(self, args, branch_arg_given):
3766         changes = []
3767
3768         if len(self.changesFile) > 0:
3769             with open(self.changesFile) as f:
3770                 output = f.readlines()
3771             changeSet = set()
3772             for line in output:
3773                 changeSet.add(int(line))
3774
3775             for change in changeSet:
3776                 changes.append(change)
3777
3778             changes.sort()
3779         else:
3780             # catch "git p4 sync" with no new branches, in a repo that
3781             # does not have any existing p4 branches
3782             if len(args) == 0:
3783                 if not self.p4BranchesInGit:
3784                     raise P4CommandException("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3785
3786                 # The default branch is master, unless --branch is used to
3787                 # specify something else.  Make sure it exists, or complain
3788                 # nicely about how to use --branch.
3789                 if not self.detectBranches:
3790                     if not branch_exists(self.branch):
3791                         if branch_arg_given:
3792                             raise P4CommandException("Error: branch %s does not exist." % self.branch)
3793                         else:
3794                             raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3795                                 self.branch)
3796
3797             if self.verbose:
3798                 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3799                                                           self.changeRange))
3800             changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3801
3802             if len(self.maxChanges) > 0:
3803                 changes = changes[:min(int(self.maxChanges), len(changes))]
3804
3805         if len(changes) == 0:
3806             if not self.silent:
3807                 print("No changes to import!")
3808         else:
3809             if not self.silent and not self.detectBranches:
3810                 print("Import destination: %s" % self.branch)
3811
3812             self.updatedBranches = set()
3813
3814             if not self.detectBranches:
3815                 if args:
3816                     # start a new branch
3817                     self.initialParent = ""
3818                 else:
3819                     # build on a previous revision
3820                     self.initialParent = parseRevision(self.branch)
3821
3822             self.importChanges(changes)
3823
3824             if not self.silent:
3825                 print("")
3826                 if len(self.updatedBranches) > 0:
3827                     sys.stdout.write("Updated branches: ")
3828                     for b in self.updatedBranches:
3829                         sys.stdout.write("%s " % b)
3830                     sys.stdout.write("\n")
3831
3832     def openStreams(self):
3833         self.importProcess = subprocess.Popen(["git", "fast-import"],
3834                                               stdin=subprocess.PIPE,
3835                                               stdout=subprocess.PIPE,
3836                                               stderr=subprocess.PIPE);
3837         self.gitOutput = self.importProcess.stdout
3838         self.gitStream = self.importProcess.stdin
3839         self.gitError = self.importProcess.stderr
3840
3841         if bytes is not str:
3842             # Wrap gitStream.write() so that it can be called using `str` arguments
3843             def make_encoded_write(write):
3844                 def encoded_write(s):
3845                     return write(s.encode() if isinstance(s, str) else s)
3846                 return encoded_write
3847
3848             self.gitStream.write = make_encoded_write(self.gitStream.write)
3849
3850     def closeStreams(self):
3851         if self.gitStream is None:
3852             return
3853         self.gitStream.close()
3854         if self.importProcess.wait() != 0:
3855             die("fast-import failed: %s" % self.gitError.read())
3856         self.gitOutput.close()
3857         self.gitError.close()
3858         self.gitStream = None
3859
3860     def run(self, args):
3861         if self.importIntoRemotes:
3862             self.refPrefix = "refs/remotes/p4/"
3863         else:
3864             self.refPrefix = "refs/heads/p4/"
3865
3866         self.sync_origin_only()
3867
3868         branch_arg_given = bool(self.branch)
3869         if len(self.branch) == 0:
3870             self.branch = self.refPrefix + "master"
3871             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3872                 system("git update-ref %s refs/heads/p4" % self.branch)
3873                 system("git branch -D p4")
3874
3875         # accept either the command-line option, or the configuration variable
3876         if self.useClientSpec:
3877             # will use this after clone to set the variable
3878             self.useClientSpec_from_options = True
3879         else:
3880             if gitConfigBool("git-p4.useclientspec"):
3881                 self.useClientSpec = True
3882         if self.useClientSpec:
3883             self.clientSpecDirs = getClientSpec()
3884
3885         # TODO: should always look at previous commits,
3886         # merge with previous imports, if possible.
3887         if args == []:
3888             if self.hasOrigin:
3889                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3890
3891             # branches holds mapping from branch name to sha1
3892             branches = p4BranchesInGit(self.importIntoRemotes)
3893
3894             # restrict to just this one, disabling detect-branches
3895             if branch_arg_given:
3896                 short = self.branch.split("/")[-1]
3897                 if short in branches:
3898                     self.p4BranchesInGit = [ short ]
3899             else:
3900                 self.p4BranchesInGit = branches.keys()
3901
3902             if len(self.p4BranchesInGit) > 1:
3903                 if not self.silent:
3904                     print("Importing from/into multiple branches")
3905                 self.detectBranches = True
3906                 for branch in branches.keys():
3907                     self.initialParents[self.refPrefix + branch] = \
3908                         branches[branch]
3909
3910             if self.verbose:
3911                 print("branches: %s" % self.p4BranchesInGit)
3912
3913             p4Change = 0
3914             for branch in self.p4BranchesInGit:
3915                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3916
3917                 settings = extractSettingsGitLog(logMsg)
3918
3919                 self.readOptions(settings)
3920                 if ('depot-paths' in settings
3921                     and 'change' in settings):
3922                     change = int(settings['change']) + 1
3923                     p4Change = max(p4Change, change)
3924
3925                     depotPaths = sorted(settings['depot-paths'])
3926                     if self.previousDepotPaths == []:
3927                         self.previousDepotPaths = depotPaths
3928                     else:
3929                         paths = []
3930                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3931                             prev_list = prev.split("/")
3932                             cur_list = cur.split("/")
3933                             for i in range(0, min(len(cur_list), len(prev_list))):
3934                                 if cur_list[i] != prev_list[i]:
3935                                     i = i - 1
3936                                     break
3937
3938                             paths.append ("/".join(cur_list[:i + 1]))
3939
3940                         self.previousDepotPaths = paths
3941
3942             if p4Change > 0:
3943                 self.depotPaths = sorted(self.previousDepotPaths)
3944                 self.changeRange = "@%s,#head" % p4Change
3945                 if not self.silent and not self.detectBranches:
3946                     print("Performing incremental import into %s git branch" % self.branch)
3947
3948         # accept multiple ref name abbreviations:
3949         #    refs/foo/bar/branch -> use it exactly
3950         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3951         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3952         if not self.branch.startswith("refs/"):
3953             if self.importIntoRemotes:
3954                 prepend = "refs/remotes/"
3955             else:
3956                 prepend = "refs/heads/"
3957             if not self.branch.startswith("p4/"):
3958                 prepend += "p4/"
3959             self.branch = prepend + self.branch
3960
3961         if len(args) == 0 and self.depotPaths:
3962             if not self.silent:
3963                 print("Depot paths: %s" % ' '.join(self.depotPaths))
3964         else:
3965             if self.depotPaths and self.depotPaths != args:
3966                 print("previous import used depot path %s and now %s was specified. "
3967                        "This doesn't work!" % (' '.join (self.depotPaths),
3968                                                ' '.join (args)))
3969                 sys.exit(1)
3970
3971             self.depotPaths = sorted(args)
3972
3973         revision = ""
3974         self.users = {}
3975
3976         # Make sure no revision specifiers are used when --changesfile
3977         # is specified.
3978         bad_changesfile = False
3979         if len(self.changesFile) > 0:
3980             for p in self.depotPaths:
3981                 if p.find("@") >= 0 or p.find("#") >= 0:
3982                     bad_changesfile = True
3983                     break
3984         if bad_changesfile:
3985             die("Option --changesfile is incompatible with revision specifiers")
3986
3987         newPaths = []
3988         for p in self.depotPaths:
3989             if p.find("@") != -1:
3990                 atIdx = p.index("@")
3991                 self.changeRange = p[atIdx:]
3992                 if self.changeRange == "@all":
3993                     self.changeRange = ""
3994                 elif ',' not in self.changeRange:
3995                     revision = self.changeRange
3996                     self.changeRange = ""
3997                 p = p[:atIdx]
3998             elif p.find("#") != -1:
3999                 hashIdx = p.index("#")
4000                 revision = p[hashIdx:]
4001                 p = p[:hashIdx]
4002             elif self.previousDepotPaths == []:
4003                 # pay attention to changesfile, if given, else import
4004                 # the entire p4 tree at the head revision
4005                 if len(self.changesFile) == 0:
4006                     revision = "#head"
4007
4008             p = re.sub ("\.\.\.$", "", p)
4009             if not p.endswith("/"):
4010                 p += "/"
4011
4012             newPaths.append(p)
4013
4014         self.depotPaths = newPaths
4015
4016         # --detect-branches may change this for each branch
4017         self.branchPrefixes = self.depotPaths
4018
4019         self.loadUserMapFromCache()
4020         self.labels = {}
4021         if self.detectLabels:
4022             self.getLabels();
4023
4024         if self.detectBranches:
4025             ## FIXME - what's a P4 projectName ?
4026             self.projectName = self.guessProjectName()
4027
4028             if self.hasOrigin:
4029                 self.getBranchMappingFromGitBranches()
4030             else:
4031                 self.getBranchMapping()
4032             if self.verbose:
4033                 print("p4-git branches: %s" % self.p4BranchesInGit)
4034                 print("initial parents: %s" % self.initialParents)
4035             for b in self.p4BranchesInGit:
4036                 if b != "master":
4037
4038                     ## FIXME
4039                     b = b[len(self.projectName):]
4040                 self.createdBranches.add(b)
4041
4042         p4_check_access()
4043
4044         self.openStreams()
4045
4046         err = None
4047
4048         try:
4049             if revision:
4050                 self.importHeadRevision(revision)
4051             else:
4052                 self.importRevisions(args, branch_arg_given)
4053
4054             if gitConfigBool("git-p4.importLabels"):
4055                 self.importLabels = True
4056
4057             if self.importLabels:
4058                 p4Labels = getP4Labels(self.depotPaths)
4059                 gitTags = getGitTags()
4060
4061                 missingP4Labels = p4Labels - gitTags
4062                 self.importP4Labels(self.gitStream, missingP4Labels)
4063
4064         except P4CommandException as e:
4065             err = e
4066
4067         finally:
4068             self.closeStreams()
4069
4070         if err:
4071             die(str(err))
4072
4073         # Cleanup temporary branches created during import
4074         if self.tempBranches != []:
4075             for branch in self.tempBranches:
4076                 read_pipe("git update-ref -d %s" % branch)
4077             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
4078
4079         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4080         # a convenient shortcut refname "p4".
4081         if self.importIntoRemotes:
4082             head_ref = self.refPrefix + "HEAD"
4083             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
4084                 system(["git", "symbolic-ref", head_ref, self.branch])
4085
4086         return True
4087
4088 class P4Rebase(Command):
4089     def __init__(self):
4090         Command.__init__(self)
4091         self.options = [
4092                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
4093         ]
4094         self.importLabels = False
4095         self.description = ("Fetches the latest revision from perforce and "
4096                             + "rebases the current work (branch) against it")
4097
4098     def run(self, args):
4099         sync = P4Sync()
4100         sync.importLabels = self.importLabels
4101         sync.run([])
4102
4103         return self.rebase()
4104
4105     def rebase(self):
4106         if os.system("git update-index --refresh") != 0:
4107             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
4108         if len(read_pipe("git diff-index HEAD --")) > 0:
4109             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
4110
4111         [upstream, settings] = findUpstreamBranchPoint()
4112         if len(upstream) == 0:
4113             die("Cannot find upstream branchpoint for rebase")
4114
4115         # the branchpoint may be p4/foo~3, so strip off the parent
4116         upstream = re.sub("~[0-9]+$", "", upstream)
4117
4118         print("Rebasing the current branch onto %s" % upstream)
4119         oldHead = read_pipe("git rev-parse HEAD").strip()
4120         system("git rebase %s" % upstream)
4121         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
4122         return True
4123
4124 class P4Clone(P4Sync):
4125     def __init__(self):
4126         P4Sync.__init__(self)
4127         self.description = "Creates a new git repository and imports from Perforce into it"
4128         self.usage = "usage: %prog [options] //depot/path[@revRange]"
4129         self.options += [
4130             optparse.make_option("--destination", dest="cloneDestination",
4131                                  action='store', default=None,
4132                                  help="where to leave result of the clone"),
4133             optparse.make_option("--bare", dest="cloneBare",
4134                                  action="store_true", default=False),
4135         ]
4136         self.cloneDestination = None
4137         self.needsGit = False
4138         self.cloneBare = False
4139
4140     def defaultDestination(self, args):
4141         ## TODO: use common prefix of args?
4142         depotPath = args[0]
4143         depotDir = re.sub("(@[^@]*)$", "", depotPath)
4144         depotDir = re.sub("(#[^#]*)$", "", depotDir)
4145         depotDir = re.sub(r"\.\.\.$", "", depotDir)
4146         depotDir = re.sub(r"/$", "", depotDir)
4147         return os.path.split(depotDir)[1]
4148
4149     def run(self, args):
4150         if len(args) < 1:
4151             return False
4152
4153         if self.keepRepoPath and not self.cloneDestination:
4154             sys.stderr.write("Must specify destination for --keep-path\n")
4155             sys.exit(1)
4156
4157         depotPaths = args
4158
4159         if not self.cloneDestination and len(depotPaths) > 1:
4160             self.cloneDestination = depotPaths[-1]
4161             depotPaths = depotPaths[:-1]
4162
4163         for p in depotPaths:
4164             if not p.startswith("//"):
4165                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
4166                 return False
4167
4168         if not self.cloneDestination:
4169             self.cloneDestination = self.defaultDestination(args)
4170
4171         print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
4172
4173         if not os.path.exists(self.cloneDestination):
4174             os.makedirs(self.cloneDestination)
4175         chdir(self.cloneDestination)
4176
4177         init_cmd = [ "git", "init" ]
4178         if self.cloneBare:
4179             init_cmd.append("--bare")
4180         retcode = subprocess.call(init_cmd)
4181         if retcode:
4182             raise CalledProcessError(retcode, init_cmd)
4183
4184         if not P4Sync.run(self, depotPaths):
4185             return False
4186
4187         # create a master branch and check out a work tree
4188         if gitBranchExists(self.branch):
4189             system([ "git", "branch", currentGitBranch(), self.branch ])
4190             if not self.cloneBare:
4191                 system([ "git", "checkout", "-f" ])
4192         else:
4193             print('Not checking out any branch, use ' \
4194                   '"git checkout -q -b master <branch>"')
4195
4196         # auto-set this variable if invoked with --use-client-spec
4197         if self.useClientSpec_from_options:
4198             system("git config --bool git-p4.useclientspec true")
4199
4200         return True
4201
4202 class P4Unshelve(Command):
4203     def __init__(self):
4204         Command.__init__(self)
4205         self.options = []
4206         self.origin = "HEAD"
4207         self.description = "Unshelve a P4 changelist into a git commit"
4208         self.usage = "usage: %prog [options] changelist"
4209         self.options += [
4210                 optparse.make_option("--origin", dest="origin",
4211                     help="Use this base revision instead of the default (%s)" % self.origin),
4212         ]
4213         self.verbose = False
4214         self.noCommit = False
4215         self.destbranch = "refs/remotes/p4-unshelved"
4216
4217     def renameBranch(self, branch_name):
4218         """ Rename the existing branch to branch_name.N
4219         """
4220
4221         found = True
4222         for i in range(0,1000):
4223             backup_branch_name = "{0}.{1}".format(branch_name, i)
4224             if not gitBranchExists(backup_branch_name):
4225                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
4226                 gitDeleteRef(branch_name)
4227                 found = True
4228                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
4229                 break
4230
4231         if not found:
4232             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
4233
4234     def findLastP4Revision(self, starting_point):
4235         """ Look back from starting_point for the first commit created by git-p4
4236             to find the P4 commit we are based on, and the depot-paths.
4237         """
4238
4239         for parent in (range(65535)):
4240             log = extractLogMessageFromGitCommit("{0}~{1}".format(starting_point, parent))
4241             settings = extractSettingsGitLog(log)
4242             if 'change' in settings:
4243                 return settings
4244
4245         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4246
4247     def createShelveParent(self, change, branch_name, sync, origin):
4248         """ Create a commit matching the parent of the shelved changelist 'change'
4249         """
4250         parent_description = p4_describe(change, shelved=True)
4251         parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4252         files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4253
4254         parent_files = []
4255         for f in files:
4256             # if it was added in the shelved changelist, it won't exist in the parent
4257             if f['action'] in self.add_actions:
4258                 continue
4259
4260             # if it was deleted in the shelved changelist it must not be deleted
4261             # in the parent - we might even need to create it if the origin branch
4262             # does not have it
4263             if f['action'] in self.delete_actions:
4264                 f['action'] = 'add'
4265
4266             parent_files.append(f)
4267
4268         sync.commit(parent_description, parent_files, branch_name,
4269                 parent=origin, allow_empty=True)
4270         print("created parent commit for {0} based on {1} in {2}".format(
4271             change, self.origin, branch_name))
4272
4273     def run(self, args):
4274         if len(args) != 1:
4275             return False
4276
4277         if not gitBranchExists(self.origin):
4278             sys.exit("origin branch {0} does not exist".format(self.origin))
4279
4280         sync = P4Sync()
4281         changes = args
4282
4283         # only one change at a time
4284         change = changes[0]
4285
4286         # if the target branch already exists, rename it
4287         branch_name = "{0}/{1}".format(self.destbranch, change)
4288         if gitBranchExists(branch_name):
4289             self.renameBranch(branch_name)
4290         sync.branch = branch_name
4291
4292         sync.verbose = self.verbose
4293         sync.suppress_meta_comment = True
4294
4295         settings = self.findLastP4Revision(self.origin)
4296         sync.depotPaths = settings['depot-paths']
4297         sync.branchPrefixes = sync.depotPaths
4298
4299         sync.openStreams()
4300         sync.loadUserMapFromCache()
4301         sync.silent = True
4302
4303         # create a commit for the parent of the shelved changelist
4304         self.createShelveParent(change, branch_name, sync, self.origin)
4305
4306         # create the commit for the shelved changelist itself
4307         description = p4_describe(change, True)
4308         files = sync.extractFilesFromCommit(description, True, change)
4309
4310         sync.commit(description, files, branch_name, "")
4311         sync.closeStreams()
4312
4313         print("unshelved changelist {0} into {1}".format(change, branch_name))
4314
4315         return True
4316
4317 class P4Branches(Command):
4318     def __init__(self):
4319         Command.__init__(self)
4320         self.options = [ ]
4321         self.description = ("Shows the git branches that hold imports and their "
4322                             + "corresponding perforce depot paths")
4323         self.verbose = False
4324
4325     def run(self, args):
4326         if originP4BranchesExist():
4327             createOrUpdateBranchesFromOrigin()
4328
4329         cmdline = "git rev-parse --symbolic "
4330         cmdline += " --remotes"
4331
4332         for line in read_pipe_lines(cmdline):
4333             line = line.strip()
4334
4335             if not line.startswith('p4/') or line == "p4/HEAD":
4336                 continue
4337             branch = line
4338
4339             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4340             settings = extractSettingsGitLog(log)
4341
4342             print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4343         return True
4344
4345 class HelpFormatter(optparse.IndentedHelpFormatter):
4346     def __init__(self):
4347         optparse.IndentedHelpFormatter.__init__(self)
4348
4349     def format_description(self, description):
4350         if description:
4351             return description + "\n"
4352         else:
4353             return ""
4354
4355 def printUsage(commands):
4356     print("usage: %s <command> [options]" % sys.argv[0])
4357     print("")
4358     print("valid commands: %s" % ", ".join(commands))
4359     print("")
4360     print("Try %s <command> --help for command specific help." % sys.argv[0])
4361     print("")
4362
4363 commands = {
4364     "debug" : P4Debug,
4365     "submit" : P4Submit,
4366     "commit" : P4Submit,
4367     "sync" : P4Sync,
4368     "rebase" : P4Rebase,
4369     "clone" : P4Clone,
4370     "rollback" : P4RollBack,
4371     "branches" : P4Branches,
4372     "unshelve" : P4Unshelve,
4373 }
4374
4375 def main():
4376     if len(sys.argv[1:]) == 0:
4377         printUsage(commands.keys())
4378         sys.exit(2)
4379
4380     cmdName = sys.argv[1]
4381     try:
4382         klass = commands[cmdName]
4383         cmd = klass()
4384     except KeyError:
4385         print("unknown command %s" % cmdName)
4386         print("")
4387         printUsage(commands.keys())
4388         sys.exit(2)
4389
4390     options = cmd.options
4391     cmd.gitdir = os.environ.get("GIT_DIR", None)
4392
4393     args = sys.argv[2:]
4394
4395     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4396     if cmd.needsGit:
4397         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4398
4399     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4400                                    options,
4401                                    description = cmd.description,
4402                                    formatter = HelpFormatter())
4403
4404     try:
4405         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4406     except:
4407         parser.print_help()
4408         raise
4409
4410     global verbose
4411     verbose = cmd.verbose
4412     if cmd.needsGit:
4413         if cmd.gitdir == None:
4414             cmd.gitdir = os.path.abspath(".git")
4415             if not isValidGitDir(cmd.gitdir):
4416                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4417                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4418                 if os.path.exists(cmd.gitdir):
4419                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4420                     if len(cdup) > 0:
4421                         chdir(cdup);
4422
4423         if not isValidGitDir(cmd.gitdir):
4424             if isValidGitDir(cmd.gitdir + "/.git"):
4425                 cmd.gitdir += "/.git"
4426             else:
4427                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4428
4429         # so git commands invoked from the P4 workspace will succeed
4430         os.environ["GIT_DIR"] = cmd.gitdir
4431
4432     if not cmd.run(args):
4433         parser.print_help()
4434         sys.exit(2)
4435
4436
4437 if __name__ == '__main__':
4438     main()