1#!/usr/bin/python
2"""Diff a repo (downstream) and its upstream.
3
4This script:
5  1. Downloads a repo source tree with specified manifest URL, branch
6     and release tag.
7  2. Retrieves the BUILD_ID from $downstream/build/core/build_id.mk.
8  3. Downloads the upstream using the BUILD_ID.
9  4. Diffs each project in these two repos.
10"""
11
12import argparse
13import datetime
14import os
15import subprocess
16import repo_diff_trees
17
18HELP_MSG = "Diff a repo (downstream) and its upstream"
19
20DOWNSTREAM_WORKSPACE = "downstream"
21UPSTREAM_WORKSPACE = "upstream"
22
23DEFAULT_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
24DEFAULT_MANIFEST_BRANCH = "android-8.0.0_r10"
25DEFAULT_UPSTREAM_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
26DEFAULT_UPSTREAM_MANIFEST_BRANCH = "android-8.0.0_r1"
27SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
28DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "android_exclusions.txt")
29
30
31def parse_args():
32  """Parse args."""
33
34  parser = argparse.ArgumentParser(description=HELP_MSG)
35
36  parser.add_argument("-u", "--manifest-url",
37                      help="manifest url",
38                      default=DEFAULT_MANIFEST_URL)
39  parser.add_argument("-b", "--manifest-branch",
40                      help="manifest branch",
41                      default=DEFAULT_MANIFEST_BRANCH)
42  parser.add_argument("-r", "--upstream-manifest-url",
43                      help="upstream manifest url",
44                      default=DEFAULT_UPSTREAM_MANIFEST_URL)
45  parser.add_argument("-a", "--upstream-manifest-branch",
46                      help="upstream manifest branch",
47                      default=DEFAULT_UPSTREAM_MANIFEST_BRANCH)
48  parser.add_argument("-e", "--exclusions-file",
49                      help="exclusions file",
50                      default=DEFAULT_EXCLUSIONS_FILE)
51  parser.add_argument("-t", "--tag",
52                      help="release tag (optional). If not set then will "
53                      "sync the latest in the branch.")
54  parser.add_argument("-i", "--ignore_error_during_sync",
55                      action="store_true",
56                      help="repo sync might fail due to varios reasons. "
57                      "Ignore these errors and move on. Use with caution.")
58
59  return parser.parse_args()
60
61
62def repo_init(url, rev, workspace):
63  """Repo init with specific url and rev.
64
65  Args:
66    url: manifest url
67    rev: manifest branch, or rev
68    workspace: the folder to init and sync code
69  """
70
71  try:
72    subprocess.check_output("repo", stderr=subprocess.PIPE,
73                            cwd=os.path.dirname(workspace), shell=True)
74  except subprocess.CalledProcessError:
75    pass
76  else:
77    raise ValueError("cannot repo-init workspace (%s), workspace is within an "
78                     "existing tree" % workspace)
79
80  print("repo init:\n  url: %s\n  rev: %s\n  workspace: %s" %
81        (url, rev, workspace))
82
83  subprocess.check_output("repo init --manifest-url=%s --manifest-branch=%s" %
84                          (url, rev), cwd=workspace, shell=True)
85
86
87def repo_sync(workspace, ignore_error, retry=5):
88  """Repo sync."""
89
90  count = 0
91  while count < retry:
92    count += 1
93    print("repo sync (retry=%d/%d):\n  workspace: %s" %
94          (count, retry, workspace))
95
96    try:
97      command = "repo sync --jobs=24 --current-branch --quiet"
98      command += " --no-tags --no-clone-bundle"
99      if ignore_error:
100        command += " --force-broken"
101      subprocess.check_output(command, cwd=workspace, shell=True)
102    except subprocess.CalledProcessError as e:
103      print "Error: %s" % e.output
104      if count == retry and not ignore_error:
105        raise e
106    # Stop retrying if the repo sync was successful
107    else:
108      break
109
110
111def get_commit_with_keyword(project_path, keyword):
112  """Get the latest commit in $project_path with the specific keyword."""
113
114  return subprocess.check_output(("git -C %s "
115                                  "rev-list --max-count=1 --grep=\"%s\" "
116                                  "HEAD") %
117                                 (project_path, keyword), shell=True).rstrip()
118
119
120def get_build_id(workspace):
121  """Get BUILD_ID defined in $workspace/build/core/build_id.mk."""
122
123  path = os.path.join(workspace, "build", "core", "build_id.mk")
124  return subprocess.check_output("source %s && echo $BUILD_ID" % path,
125                                 shell=True).rstrip()
126
127
128def repo_sync_specific_release(url, branch, tag, workspace, ignore_error):
129  """Repo sync source with the specific release tag."""
130
131  if not os.path.exists(workspace):
132    os.makedirs(workspace)
133
134  manifest_path = os.path.join(workspace, ".repo", "manifests")
135
136  repo_init(url, branch, workspace)
137
138  if tag:
139    rev = get_commit_with_keyword(manifest_path, tag)
140    if not rev:
141      raise(ValueError("could not find a manifest revision for tag " + tag))
142    repo_init(url, rev, workspace)
143
144  repo_sync(workspace, ignore_error)
145
146
147def diff(manifest_url, manifest_branch, tag,
148         upstream_manifest_url, upstream_manifest_branch,
149         exclusions_file, ignore_error_during_sync):
150  """Syncs and diffs an Android workspace against an upstream workspace."""
151
152  workspace = os.path.abspath(DOWNSTREAM_WORKSPACE)
153  upstream_workspace = os.path.abspath(UPSTREAM_WORKSPACE)
154  # repo sync downstream source tree
155  repo_sync_specific_release(
156      manifest_url,
157      manifest_branch,
158      tag,
159      workspace,
160      ignore_error_during_sync)
161
162  build_id = None
163
164  if tag:
165    # get the build_id so that we know which rev of upstream we need
166    build_id = get_build_id(workspace)
167    if not build_id:
168      raise(ValueError("Error: could not find the Build ID of " + workspace))
169
170  # repo sync upstream source tree
171  repo_sync_specific_release(
172      upstream_manifest_url,
173      upstream_manifest_branch,
174      build_id,
175      upstream_workspace,
176      ignore_error_during_sync)
177
178
179  # make output folder
180  if tag:
181    output_folder = os.path.abspath(tag.replace(" ", "_"))
182  else:
183    current_time = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
184    output_folder = os.path.abspath(current_time)
185
186  if not os.path.exists(output_folder):
187      os.makedirs(output_folder)
188
189  # do the comparison
190  repo_diff_trees.diff(
191      upstream_workspace,
192      workspace,
193      os.path.join(output_folder, "project.csv"),
194      os.path.join(output_folder, "commit.csv"),
195      os.path.abspath(exclusions_file),
196  )
197
198
199def main():
200  args = parse_args()
201
202  diff(args.manifest_url,
203       args.manifest_branch,
204       args.tag,
205       args.upstream_manifest_url,
206       args.upstream_manifest_branch,
207       args.exclusions_file,
208       args.ignore_error_during_sync)
209
210if __name__ == "__main__":
211  main()
212