• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
4#
5#                     The LLVM Compiler Infrastructure
6#
7# This file is distributed under the University of Illinois Open Source
8# License. See LICENSE.TXT for details.
9#
10# ==------------------------------------------------------------------------==#
11
12"""
13git-llvm integration
14====================
15
16This file provides integration for git.
17"""
18
19from __future__ import print_function
20import argparse
21import collections
22import contextlib
23import errno
24import os
25import re
26import subprocess
27import sys
28import tempfile
29import time
30assert sys.version_info >= (2, 7)
31
32
33# It's *almost* a straightforward mapping from the monorepo to svn...
34GIT_TO_SVN_DIR = {
35    d: (d + '/trunk')
36    for d in [
37        'clang-tools-extra',
38        'compiler-rt',
39        'debuginfo-tests',
40        'dragonegg',
41        'klee',
42        'libclc',
43        'libcxx',
44        'libcxxabi',
45        'libunwind',
46        'lld',
47        'lldb',
48        'llgo',
49        'llvm',
50        'openmp',
51        'parallel-libs',
52        'polly',
53    ]
54}
55GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
56
57VERBOSE = False
58QUIET = False
59dev_null_fd = None
60
61
62def eprint(*args, **kwargs):
63    print(*args, file=sys.stderr, **kwargs)
64
65
66def log(*args, **kwargs):
67    if QUIET:
68        return
69    print(*args, **kwargs)
70
71
72def log_verbose(*args, **kwargs):
73    if not VERBOSE:
74        return
75    print(*args, **kwargs)
76
77
78def die(msg):
79    eprint(msg)
80    sys.exit(1)
81
82
83def first_dirname(d):
84    while True:
85        (head, tail) = os.path.split(d)
86        if not head or head == '/':
87            return tail
88        d = head
89
90
91def get_dev_null():
92    """Lazily create a /dev/null fd for use in shell()"""
93    global dev_null_fd
94    if dev_null_fd is None:
95        dev_null_fd = open(os.devnull, 'w')
96    return dev_null_fd
97
98
99def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
100          ignore_errors=False):
101    log_verbose('Running: %s' % ' '.join(cmd))
102
103    err_pipe = subprocess.PIPE
104    if ignore_errors:
105        # Silence errors if requested.
106        err_pipe = get_dev_null()
107
108    start = time.time()
109    p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
110                         stdin=subprocess.PIPE, universal_newlines=True)
111    stdout, stderr = p.communicate(input=stdin)
112    elapsed = time.time() - start
113
114    log_verbose('Command took %0.1fs' % elapsed)
115
116    if p.returncode == 0 or ignore_errors:
117        if stderr and not ignore_errors:
118            eprint('`%s` printed to stderr:' % ' '.join(cmd))
119            eprint(stderr.rstrip())
120        if strip:
121            stdout = stdout.rstrip('\r\n')
122        return stdout
123    err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode)
124    eprint(err_msg)
125    if stderr:
126        eprint(stderr.rstrip())
127    if die_on_failure:
128        sys.exit(2)
129    raise RuntimeError(err_msg)
130
131
132def git(*cmd, **kwargs):
133    return shell(['git'] + list(cmd), kwargs.get('strip', True))
134
135
136def svn(cwd, *cmd, **kwargs):
137    # TODO: Better way to do default arg when we have *cmd?
138    return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None),
139                 ignore_errors=kwargs.get('ignore_errors', None))
140
141def program_exists(cmd):
142    if sys.platform == 'win32' and not cmd.endswith('.exe'):
143        cmd += '.exe'
144    for path in os.environ["PATH"].split(os.pathsep):
145        if os.access(os.path.join(path, cmd), os.X_OK):
146            return True
147    return False
148
149def get_default_rev_range():
150    # Get the branch tracked by the current branch, as set by
151    # git branch --set-upstream-to  See http://serverfault.com/a/352236/38694.
152    cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
153    upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
154                          cur_branch)
155    if not upstream_branch:
156        upstream_branch = 'origin/master'
157
158    # Get the newest common ancestor between HEAD and our upstream branch.
159    upstream_rev = git('merge-base', 'HEAD', upstream_branch)
160    return '%s..' % upstream_rev
161
162
163def get_revs_to_push(rev_range):
164    if not rev_range:
165        rev_range = get_default_rev_range()
166    # Use git show rather than some plumbing command to figure out which revs
167    # are in rev_range because it handles single revs (HEAD^) and ranges
168    # (foo..bar) like we want.
169    revs = git('show', '--reverse', '--quiet',
170               '--pretty=%h', rev_range).splitlines()
171    if not revs:
172        die('Nothing to push: No revs in range %s.' % rev_range)
173    return revs
174
175
176def clean_and_update_svn(svn_repo):
177    svn(svn_repo, 'revert', '-R', '.')
178
179    # Unfortunately it appears there's no svn equivalent for git clean, so we
180    # have to do it ourselves.
181    for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
182        if not line.startswith('?'):
183            continue
184        filename = line[1:].strip()
185        os.remove(os.path.join(svn_repo, filename))
186
187    svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values()))
188
189
190def svn_init(svn_root):
191    if not os.path.exists(svn_root):
192        log('Creating svn staging directory: (%s)' % (svn_root))
193        os.makedirs(svn_root)
194        log('This is a one-time initialization, please be patient for a few'
195            ' minutes...')
196        svn(svn_root, 'checkout', '--depth=immediates',
197            'https://llvm.org/svn/llvm-project/', '.')
198        svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values()))
199        log("svn staging area ready in '%s'" % svn_root)
200    if not os.path.isdir(svn_root):
201        die("Can't initialize svn staging dir (%s)" % svn_root)
202
203
204def fix_eol_style_native(rev, sr, svn_sr_path):
205    """Fix line endings before applying patches with Unix endings
206
207    SVN on Windows will check out files with CRLF for files with the
208    svn:eol-style property set to "native". This breaks `git apply`, which
209    typically works with Unix-line ending patches. Work around the problem here
210    by doing a dos2unix up front for files with svn:eol-style set to "native".
211    SVN will not commit a mass line ending re-doing because it detects the line
212    ending format for files with this property.
213    """
214    files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--',
215                sr).split('\n')
216    files = [f.split('/', 1)[1] for f in files]
217    # Skip files that don't exist in SVN yet.
218    files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
219    # Use ignore_errors because 'svn propget' prints errors if the file doesn't
220    # have the named property. There doesn't seem to be a way to suppress that.
221    eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
222                    ignore_errors=True)
223    crlf_files = []
224    if len(files) == 1:
225        # No need to split propget output on ' - ' when we have one file.
226        if eol_props.strip() == 'native':
227            crlf_files = files
228    else:
229        for eol_prop in eol_props.split('\n'):
230            # Remove spare CR.
231            eol_prop = eol_prop.strip('\r')
232            if not eol_prop:
233                continue
234            prop_parts = eol_prop.rsplit(' - ', 1)
235            if len(prop_parts) != 2:
236                eprint("unable to parse svn propget line:")
237                eprint(eol_prop)
238                continue
239            (f, eol_style) = prop_parts
240            if eol_style == 'native':
241                crlf_files.append(f)
242    # Reformat all files with native SVN line endings to Unix format. SVN knows
243    # files with native line endings are text files. It will commit just the
244    # diff, and not a mass line ending change.
245    shell(['dos2unix', '-q'] + crlf_files, cwd=svn_sr_path)
246
247
248def svn_push_one_rev(svn_repo, rev, dry_run):
249    files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
250                rev).split('\n')
251    subrepos = {first_dirname(f) for f in files}
252    if not subrepos:
253        raise RuntimeError('Empty diff for rev %s?' % rev)
254
255    status = svn(svn_repo, 'status', '--no-ignore')
256    if status:
257        die("Can't push git rev %s because svn status is not empty:\n%s" %
258            (rev, status))
259
260    for sr in subrepos:
261        svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
262        if os.name == 'nt':
263            fix_eol_style_native(rev, sr, svn_sr_path)
264        diff = git('show', '--binary', rev, '--', sr, strip=False)
265        # git is the only thing that can handle its own patches...
266        log_verbose('Apply patch: %s' % diff)
267        try:
268            shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff,
269                  die_on_failure=False)
270        except RuntimeError as e:
271            eprint("Patch doesn't apply: maybe you should try `git pull -r` "
272                   "first?")
273            sys.exit(2)
274
275    status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
276
277    for l in (l for l in status_lines if (l.startswith('?') or
278                                          l.startswith('I'))):
279        svn(svn_repo, 'add', '--no-ignore', l[1:].strip())
280    for l in (l for l in status_lines if l.startswith('!')):
281        svn(svn_repo, 'remove', l[1:].strip())
282
283    # Now we're ready to commit.
284    commit_msg = git('show', '--pretty=%B', '--quiet', rev)
285    if not dry_run:
286        log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive'))
287        log('Committed %s to svn.' % rev)
288    else:
289        log("Would have committed %s to svn, if this weren't a dry run." % rev)
290
291
292def cmd_push(args):
293    '''Push changes back to SVN: this is extracted from Justin Lebar's script
294    available here: https://github.com/jlebar/llvm-repo-tools/
295
296    Note: a current limitation is that git does not track file rename, so they
297    will show up in SVN as delete+add.
298    '''
299    # Get the git root
300    git_root = git('rev-parse', '--show-toplevel')
301    if not os.path.isdir(git_root):
302        die("Can't find git root dir")
303
304    # Push from the root of the git repo
305    os.chdir(git_root)
306
307    # We need a staging area for SVN, let's hide it in the .git directory.
308    dot_git_dir = git('rev-parse', '--git-common-dir')
309    svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn')
310    svn_init(svn_root)
311
312    rev_range = args.rev_range
313    dry_run = args.dry_run
314    revs = get_revs_to_push(rev_range)
315    log('Pushing %d commit%s:\n%s' %
316        (len(revs), 's' if len(revs) != 1
317         else '', '\n'.join('  ' + git('show', '--oneline', '--quiet', c)
318                            for c in revs)))
319    for r in revs:
320        clean_and_update_svn(svn_root)
321        svn_push_one_rev(svn_root, r, dry_run)
322
323
324if __name__ == '__main__':
325    if not program_exists('svn'):
326        die('error: git-llvm needs svn command, but svn is not installed.')
327
328    argv = sys.argv[1:]
329    p = argparse.ArgumentParser(
330        prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter,
331        description=__doc__)
332    subcommands = p.add_subparsers(title='subcommands',
333                                   description='valid subcommands',
334                                   help='additional help')
335    verbosity_group = p.add_mutually_exclusive_group()
336    verbosity_group.add_argument('-q', '--quiet', action='store_true',
337                                 help='print less information')
338    verbosity_group.add_argument('-v', '--verbose', action='store_true',
339                                 help='print more information')
340
341    parser_push = subcommands.add_parser(
342        'push', description=cmd_push.__doc__,
343        help='push changes back to the LLVM SVN repository')
344    parser_push.add_argument(
345        '-n',
346        '--dry-run',
347        dest='dry_run',
348        action='store_true',
349        help='Do everything other than commit to svn.  Leaves junk in the svn '
350        'repo, so probably will not work well if you try to commit more '
351        'than one rev.')
352    parser_push.add_argument(
353        'rev_range',
354        metavar='GIT_REVS',
355        type=str,
356        nargs='?',
357        help="revs to push (default: everything not in the branch's "
358        'upstream, or not in origin/master if the branch lacks '
359        'an explicit upstream)')
360    parser_push.set_defaults(func=cmd_push)
361    args = p.parse_args(argv)
362    VERBOSE = args.verbose
363    QUIET = args.quiet
364
365    # Dispatch to the right subcommand
366    args.func(args)
367