1#!/usr/bin/env python 2# 3# ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========# 4# 5# The LLVM Compiler Infrastructure 6# 7# This file is distributed under the University of Illinois Open Source 8# License. See LICENSE.TXT for details. 9# 10# ==------------------------------------------------------------------------==# 11 12""" 13git-llvm integration 14==================== 15 16This file provides integration for git. 17""" 18 19from __future__ import print_function 20import argparse 21import collections 22import contextlib 23import errno 24import os 25import re 26import subprocess 27import sys 28import tempfile 29import time 30assert sys.version_info >= (2, 7) 31 32 33# It's *almost* a straightforward mapping from the monorepo to svn... 34GIT_TO_SVN_DIR = { 35 d: (d + '/trunk') 36 for d in [ 37 'clang-tools-extra', 38 'compiler-rt', 39 'debuginfo-tests', 40 'dragonegg', 41 'klee', 42 'libclc', 43 'libcxx', 44 'libcxxabi', 45 'libunwind', 46 'lld', 47 'lldb', 48 'llgo', 49 'llvm', 50 'openmp', 51 'parallel-libs', 52 'polly', 53 ] 54} 55GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'}) 56 57VERBOSE = False 58QUIET = False 59dev_null_fd = None 60 61 62def eprint(*args, **kwargs): 63 print(*args, file=sys.stderr, **kwargs) 64 65 66def log(*args, **kwargs): 67 if QUIET: 68 return 69 print(*args, **kwargs) 70 71 72def log_verbose(*args, **kwargs): 73 if not VERBOSE: 74 return 75 print(*args, **kwargs) 76 77 78def die(msg): 79 eprint(msg) 80 sys.exit(1) 81 82 83def first_dirname(d): 84 while True: 85 (head, tail) = os.path.split(d) 86 if not head or head == '/': 87 return tail 88 d = head 89 90 91def get_dev_null(): 92 """Lazily create a /dev/null fd for use in shell()""" 93 global dev_null_fd 94 if dev_null_fd is None: 95 dev_null_fd = open(os.devnull, 'w') 96 return dev_null_fd 97 98 99def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True, 100 ignore_errors=False): 101 log_verbose('Running: %s' % ' '.join(cmd)) 102 103 err_pipe = subprocess.PIPE 104 if ignore_errors: 105 # Silence errors if requested. 106 err_pipe = get_dev_null() 107 108 start = time.time() 109 p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe, 110 stdin=subprocess.PIPE, universal_newlines=True) 111 stdout, stderr = p.communicate(input=stdin) 112 elapsed = time.time() - start 113 114 log_verbose('Command took %0.1fs' % elapsed) 115 116 if p.returncode == 0 or ignore_errors: 117 if stderr and not ignore_errors: 118 eprint('`%s` printed to stderr:' % ' '.join(cmd)) 119 eprint(stderr.rstrip()) 120 if strip: 121 stdout = stdout.rstrip('\r\n') 122 return stdout 123 err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode) 124 eprint(err_msg) 125 if stderr: 126 eprint(stderr.rstrip()) 127 if die_on_failure: 128 sys.exit(2) 129 raise RuntimeError(err_msg) 130 131 132def git(*cmd, **kwargs): 133 return shell(['git'] + list(cmd), kwargs.get('strip', True)) 134 135 136def svn(cwd, *cmd, **kwargs): 137 # TODO: Better way to do default arg when we have *cmd? 138 return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None), 139 ignore_errors=kwargs.get('ignore_errors', None)) 140 141def program_exists(cmd): 142 if sys.platform == 'win32' and not cmd.endswith('.exe'): 143 cmd += '.exe' 144 for path in os.environ["PATH"].split(os.pathsep): 145 if os.access(os.path.join(path, cmd), os.X_OK): 146 return True 147 return False 148 149def get_default_rev_range(): 150 # Get the branch tracked by the current branch, as set by 151 # git branch --set-upstream-to See http://serverfault.com/a/352236/38694. 152 cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD') 153 upstream_branch = git('for-each-ref', '--format=%(upstream:short)', 154 cur_branch) 155 if not upstream_branch: 156 upstream_branch = 'origin/master' 157 158 # Get the newest common ancestor between HEAD and our upstream branch. 159 upstream_rev = git('merge-base', 'HEAD', upstream_branch) 160 return '%s..' % upstream_rev 161 162 163def get_revs_to_push(rev_range): 164 if not rev_range: 165 rev_range = get_default_rev_range() 166 # Use git show rather than some plumbing command to figure out which revs 167 # are in rev_range because it handles single revs (HEAD^) and ranges 168 # (foo..bar) like we want. 169 revs = git('show', '--reverse', '--quiet', 170 '--pretty=%h', rev_range).splitlines() 171 if not revs: 172 die('Nothing to push: No revs in range %s.' % rev_range) 173 return revs 174 175 176def clean_and_update_svn(svn_repo): 177 svn(svn_repo, 'revert', '-R', '.') 178 179 # Unfortunately it appears there's no svn equivalent for git clean, so we 180 # have to do it ourselves. 181 for line in svn(svn_repo, 'status', '--no-ignore').split('\n'): 182 if not line.startswith('?'): 183 continue 184 filename = line[1:].strip() 185 os.remove(os.path.join(svn_repo, filename)) 186 187 svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values())) 188 189 190def svn_init(svn_root): 191 if not os.path.exists(svn_root): 192 log('Creating svn staging directory: (%s)' % (svn_root)) 193 os.makedirs(svn_root) 194 log('This is a one-time initialization, please be patient for a few' 195 ' minutes...') 196 svn(svn_root, 'checkout', '--depth=immediates', 197 'https://llvm.org/svn/llvm-project/', '.') 198 svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values())) 199 log("svn staging area ready in '%s'" % svn_root) 200 if not os.path.isdir(svn_root): 201 die("Can't initialize svn staging dir (%s)" % svn_root) 202 203 204def fix_eol_style_native(rev, sr, svn_sr_path): 205 """Fix line endings before applying patches with Unix endings 206 207 SVN on Windows will check out files with CRLF for files with the 208 svn:eol-style property set to "native". This breaks `git apply`, which 209 typically works with Unix-line ending patches. Work around the problem here 210 by doing a dos2unix up front for files with svn:eol-style set to "native". 211 SVN will not commit a mass line ending re-doing because it detects the line 212 ending format for files with this property. 213 """ 214 files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--', 215 sr).split('\n') 216 files = [f.split('/', 1)[1] for f in files] 217 # Skip files that don't exist in SVN yet. 218 files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))] 219 # Use ignore_errors because 'svn propget' prints errors if the file doesn't 220 # have the named property. There doesn't seem to be a way to suppress that. 221 eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files, 222 ignore_errors=True) 223 crlf_files = [] 224 if len(files) == 1: 225 # No need to split propget output on ' - ' when we have one file. 226 if eol_props.strip() == 'native': 227 crlf_files = files 228 else: 229 for eol_prop in eol_props.split('\n'): 230 # Remove spare CR. 231 eol_prop = eol_prop.strip('\r') 232 if not eol_prop: 233 continue 234 prop_parts = eol_prop.rsplit(' - ', 1) 235 if len(prop_parts) != 2: 236 eprint("unable to parse svn propget line:") 237 eprint(eol_prop) 238 continue 239 (f, eol_style) = prop_parts 240 if eol_style == 'native': 241 crlf_files.append(f) 242 # Reformat all files with native SVN line endings to Unix format. SVN knows 243 # files with native line endings are text files. It will commit just the 244 # diff, and not a mass line ending change. 245 shell(['dos2unix', '-q'] + crlf_files, cwd=svn_sr_path) 246 247 248def svn_push_one_rev(svn_repo, rev, dry_run): 249 files = git('diff-tree', '--no-commit-id', '--name-only', '-r', 250 rev).split('\n') 251 subrepos = {first_dirname(f) for f in files} 252 if not subrepos: 253 raise RuntimeError('Empty diff for rev %s?' % rev) 254 255 status = svn(svn_repo, 'status', '--no-ignore') 256 if status: 257 die("Can't push git rev %s because svn status is not empty:\n%s" % 258 (rev, status)) 259 260 for sr in subrepos: 261 svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr]) 262 if os.name == 'nt': 263 fix_eol_style_native(rev, sr, svn_sr_path) 264 diff = git('show', '--binary', rev, '--', sr, strip=False) 265 # git is the only thing that can handle its own patches... 266 log_verbose('Apply patch: %s' % diff) 267 try: 268 shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff, 269 die_on_failure=False) 270 except RuntimeError as e: 271 eprint("Patch doesn't apply: maybe you should try `git pull -r` " 272 "first?") 273 sys.exit(2) 274 275 status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n') 276 277 for l in (l for l in status_lines if (l.startswith('?') or 278 l.startswith('I'))): 279 svn(svn_repo, 'add', '--no-ignore', l[1:].strip()) 280 for l in (l for l in status_lines if l.startswith('!')): 281 svn(svn_repo, 'remove', l[1:].strip()) 282 283 # Now we're ready to commit. 284 commit_msg = git('show', '--pretty=%B', '--quiet', rev) 285 if not dry_run: 286 log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive')) 287 log('Committed %s to svn.' % rev) 288 else: 289 log("Would have committed %s to svn, if this weren't a dry run." % rev) 290 291 292def cmd_push(args): 293 '''Push changes back to SVN: this is extracted from Justin Lebar's script 294 available here: https://github.com/jlebar/llvm-repo-tools/ 295 296 Note: a current limitation is that git does not track file rename, so they 297 will show up in SVN as delete+add. 298 ''' 299 # Get the git root 300 git_root = git('rev-parse', '--show-toplevel') 301 if not os.path.isdir(git_root): 302 die("Can't find git root dir") 303 304 # Push from the root of the git repo 305 os.chdir(git_root) 306 307 # We need a staging area for SVN, let's hide it in the .git directory. 308 dot_git_dir = git('rev-parse', '--git-common-dir') 309 svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn') 310 svn_init(svn_root) 311 312 rev_range = args.rev_range 313 dry_run = args.dry_run 314 revs = get_revs_to_push(rev_range) 315 log('Pushing %d commit%s:\n%s' % 316 (len(revs), 's' if len(revs) != 1 317 else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c) 318 for c in revs))) 319 for r in revs: 320 clean_and_update_svn(svn_root) 321 svn_push_one_rev(svn_root, r, dry_run) 322 323 324if __name__ == '__main__': 325 if not program_exists('svn'): 326 die('error: git-llvm needs svn command, but svn is not installed.') 327 328 argv = sys.argv[1:] 329 p = argparse.ArgumentParser( 330 prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter, 331 description=__doc__) 332 subcommands = p.add_subparsers(title='subcommands', 333 description='valid subcommands', 334 help='additional help') 335 verbosity_group = p.add_mutually_exclusive_group() 336 verbosity_group.add_argument('-q', '--quiet', action='store_true', 337 help='print less information') 338 verbosity_group.add_argument('-v', '--verbose', action='store_true', 339 help='print more information') 340 341 parser_push = subcommands.add_parser( 342 'push', description=cmd_push.__doc__, 343 help='push changes back to the LLVM SVN repository') 344 parser_push.add_argument( 345 '-n', 346 '--dry-run', 347 dest='dry_run', 348 action='store_true', 349 help='Do everything other than commit to svn. Leaves junk in the svn ' 350 'repo, so probably will not work well if you try to commit more ' 351 'than one rev.') 352 parser_push.add_argument( 353 'rev_range', 354 metavar='GIT_REVS', 355 type=str, 356 nargs='?', 357 help="revs to push (default: everything not in the branch's " 358 'upstream, or not in origin/master if the branch lacks ' 359 'an explicit upstream)') 360 parser_push.set_defaults(func=cmd_push) 361 args = p.parse_args(argv) 362 VERBOSE = args.verbose 363 QUIET = args.quiet 364 365 # Dispatch to the right subcommand 366 args.func(args) 367