1#!/usr/bin/env python
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15""" Mirrors a Gerrit repo into GitHub.
16
17Mirrors all the branches (refs/heads/foo) from Gerrit to Github as-is, taking
18care of propagating also deletions.
19
20This script used to be more complex, turning all the Gerrit CLs
21(refs/changes/NN/cl_number/patchset_number) into Github branches
22(refs/heads/cl_number). This use case was dropped as we moved away from Travis.
23See the git history of this file for more.
24"""
25
26import argparse
27import logging
28import os
29import re
30import shutil
31import subprocess
32import sys
33import time
34
35CUR_DIR = os.path.dirname(os.path.abspath(__file__))
36GIT_UPSTREAM = 'https://android.googlesource.com/platform/external/perfetto/'
37GIT_MIRROR = 'git@github.com:google/perfetto.git'
38WORKDIR = os.path.join(CUR_DIR, 'repo')
39
40# Min delay (in seconds) between two consecutive git poll cycles. This is to
41# avoid hitting gerrit API quota limits.
42POLL_PERIOD_SEC = 60
43
44# The actual key is stored into the Google Cloud project metadata.
45ENV = {'GIT_SSH_COMMAND': 'ssh -i ' + os.path.join(CUR_DIR, 'deploy_key')}
46
47
48def GitCmd(*args, **kwargs):
49  cmd = ['git'] + list(args)
50  p = subprocess.Popen(
51      cmd,
52      stdin=subprocess.PIPE,
53      stdout=subprocess.PIPE,
54      stderr=sys.stderr,
55      cwd=WORKDIR,
56      env=ENV)
57  out = p.communicate(kwargs.get('stdin'))[0]
58  assert p.returncode == 0, 'FAIL: ' + ' '.join(cmd)
59  return out
60
61
62# Create a git repo that mirrors both the upstream and the mirror repos.
63def Setup(args):
64  if os.path.exists(WORKDIR):
65    if args.no_clean:
66      return
67    shutil.rmtree(WORKDIR)
68  os.makedirs(WORKDIR)
69  GitCmd('init', '--bare', '--quiet')
70  GitCmd('remote', 'add', 'upstream', GIT_UPSTREAM)
71  GitCmd('config', 'remote.upstream.tagOpt', '--no-tags')
72  GitCmd('config', '--add', 'remote.upstream.fetch',
73         '+refs/heads/*:refs/remotes/upstream/heads/*')
74  GitCmd('config', '--add', 'remote.upstream.fetch',
75         '+refs/tags/*:refs/remotes/upstream/tags/*')
76  GitCmd('remote', 'add', 'mirror', GIT_MIRROR, '--mirror=fetch')
77
78
79def Sync(args):
80  logging.info('Fetching git remotes')
81  GitCmd('fetch', '--all', '--quiet')
82  all_refs = GitCmd('show-ref')
83  future_heads = {}
84  current_heads = {}
85
86  # List all refs from both repos and:
87  # 1. Keep track of all branch heads refnames and sha1s from the (github)
88  #    mirror into |current_heads|.
89  # 2. Keep track of all upstream (AOSP) branch heads into |future_heads|. Note:
90  #    this includes only pure branches and NOT CLs. CLs and their patchsets are
91  #    stored in a hidden ref (refs/changes) which is NOT under refs/heads.
92  # 3. Keep track of all upstream (AOSP) CLs from the refs/changes namespace
93  #    into changes[cl_number][patchset_number].
94  for line in all_refs.splitlines():
95    ref_sha1, ref = line.split()
96
97    FILTER_REGEX = r'(heads/master|heads/releases/.*|tags/v\d+\.\d+)$'
98    m = re.match('refs/' + FILTER_REGEX, ref)
99    if m is not None:
100      branch = m.group(1)
101      current_heads['refs/' + branch] = ref_sha1
102      continue
103
104    m = re.match('refs/remotes/upstream/' + FILTER_REGEX, ref)
105    if m is not None:
106      branch = m.group(1)
107      future_heads['refs/' + branch] = ref_sha1
108      continue
109
110  deleted_heads = set(current_heads) - set(future_heads)
111  logging.info('current_heads: %d, future_heads: %d, deleted_heads: %d',
112               len(current_heads), len(future_heads), len(deleted_heads))
113
114  # Now compute:
115  # 1. The set of branches in the mirror (github) that have been deleted on the
116  #    upstream (AOSP) repo. These will be deleted also from the mirror.
117  # 2. The set of rewritten branches to be updated.
118  update_ref_cmd = ''
119  for ref_to_delete in deleted_heads:
120    update_ref_cmd += 'delete %s\n' % ref_to_delete
121  for ref_to_update, ref_sha1 in future_heads.iteritems():
122    if current_heads.get(ref_to_update) != ref_sha1:
123      update_ref_cmd += 'update %s %s\n' % (ref_to_update, ref_sha1)
124
125  GitCmd('update-ref', '--stdin', stdin=update_ref_cmd)
126
127  if args.push:
128    logging.info('Pushing updates')
129    GitCmd('push', 'mirror', '--all', '--prune', '--force', '--follow-tags')
130    GitCmd('gc', '--prune=all', '--aggressive', '--quiet')
131  else:
132    logging.info('Dry-run mode, skipping git push. Pass --push for prod mode.')
133
134
135def Main():
136  parser = argparse.ArgumentParser()
137  parser.add_argument('--push', default=False, action='store_true')
138  parser.add_argument('--no-clean', default=False, action='store_true')
139  parser.add_argument('-v', dest='verbose', default=False, action='store_true')
140  args = parser.parse_args()
141
142  logging.basicConfig(
143      format='%(asctime)s %(levelname)-8s %(message)s',
144      level=logging.DEBUG if args.verbose else logging.INFO,
145      datefmt='%Y-%m-%d %H:%M:%S')
146
147  logging.info('Setting up git repo one-off')
148  Setup(args)
149  while True:
150    logging.info('------- BEGINNING OF SYNC CYCLE -------')
151    Sync(args)
152    logging.info('------- END OF SYNC CYCLE -------')
153    time.sleep(POLL_PERIOD_SEC)
154
155
156if __name__ == '__main__':
157  sys.exit(Main())
158