1#!/usr/bin/env python
2# Copyright 2014 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# This script retrieves the history of all V8 branches and
7# their corresponding Chromium revisions.
8
9# Requires a chromium checkout with branch heads:
10# gclient sync --with_branch_heads
11# gclient fetch
12
13import argparse
14import csv
15import itertools
16import json
17import os
18import re
19import sys
20
21from common_includes import *
22
23CONFIG = {
24  "BRANCHNAME": "retrieve-v8-releases",
25  "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile",
26}
27
28# Expression for retrieving the bleeding edge revision from a commit message.
29PUSH_MSG_SVN_RE = re.compile(r".* \(based on bleeding_edge revision r(\d+)\)$")
30PUSH_MSG_GIT_RE = re.compile(r".* \(based on ([a-fA-F0-9]+)\)$")
31
32# Expression for retrieving the merged patches from a merge commit message
33# (old and new format).
34MERGE_MESSAGE_RE = re.compile(r"^.*[M|m]erged (.+)(\)| into).*$", re.M)
35
36CHERRY_PICK_TITLE_GIT_RE = re.compile(r"^.* \(cherry\-pick\)\.?$")
37
38# New git message for cherry-picked CLs. One message per line.
39MERGE_MESSAGE_GIT_RE = re.compile(r"^Merged ([a-fA-F0-9]+)\.?$")
40
41# Expression for retrieving reverted patches from a commit message (old and
42# new format).
43ROLLBACK_MESSAGE_RE = re.compile(r"^.*[R|r]ollback of (.+)(\)| in).*$", re.M)
44
45# New git message for reverted CLs. One message per line.
46ROLLBACK_MESSAGE_GIT_RE = re.compile(r"^Rollback of ([a-fA-F0-9]+)\.?$")
47
48# Expression for retrieving the code review link.
49REVIEW_LINK_RE = re.compile(r"^Review URL: (.+)$", re.M)
50
51# Expression with three versions (historical) for extracting the v8 revision
52# from the chromium DEPS file.
53DEPS_RE = re.compile(r"""^\s*(?:["']v8_revision["']: ["']"""
54                     """|\(Var\("googlecode_url"\) % "v8"\) \+ "\/trunk@"""
55                     """|"http\:\/\/v8\.googlecode\.com\/svn\/trunk@)"""
56                     """([^"']+)["'].*$""", re.M)
57
58# Expression to pick tag and revision for bleeding edge tags. To be used with
59# output of 'svn log'.
60BLEEDING_EDGE_TAGS_RE = re.compile(
61    r"A \/tags\/([^\s]+) \(from \/branches\/bleeding_edge\:(\d+)\)")
62
63OMAHA_PROXY_URL = "http://omahaproxy.appspot.com/"
64
65def SortBranches(branches):
66  """Sort branches with version number names."""
67  return sorted(branches, key=SortingKey, reverse=True)
68
69
70def FilterDuplicatesAndReverse(cr_releases):
71  """Returns the chromium releases in reverse order filtered by v8 revision
72  duplicates.
73
74  cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev.
75  """
76  last = ""
77  result = []
78  for release in reversed(cr_releases):
79    if last == release[1]:
80      continue
81    last = release[1]
82    result.append(release)
83  return result
84
85
86def BuildRevisionRanges(cr_releases):
87  """Returns a mapping of v8 revision -> chromium ranges.
88  The ranges are comma-separated, each range has the form R1:R2. The newest
89  entry is the only one of the form R1, as there is no end range.
90
91  cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev.
92  cr_rev either refers to a chromium commit position or a chromium branch
93  number.
94  """
95  range_lists = {}
96  cr_releases = FilterDuplicatesAndReverse(cr_releases)
97
98  # Visit pairs of cr releases from oldest to newest.
99  for cr_from, cr_to in itertools.izip(
100      cr_releases, itertools.islice(cr_releases, 1, None)):
101
102    # Assume the chromium revisions are all different.
103    assert cr_from[0] != cr_to[0]
104
105    ran = "%s:%d" % (cr_from[0], int(cr_to[0]) - 1)
106
107    # Collect the ranges in lists per revision.
108    range_lists.setdefault(cr_from[1], []).append(ran)
109
110  # Add the newest revision.
111  if cr_releases:
112    range_lists.setdefault(cr_releases[-1][1], []).append(cr_releases[-1][0])
113
114  # Stringify and comma-separate the range lists.
115  return dict((hsh, ", ".join(ran)) for hsh, ran in range_lists.iteritems())
116
117
118def MatchSafe(match):
119  if match:
120    return match.group(1)
121  else:
122    return ""
123
124
125class Preparation(Step):
126  MESSAGE = "Preparation."
127
128  def RunStep(self):
129    self.CommonPrepare()
130    self.PrepareBranch()
131
132
133class RetrieveV8Releases(Step):
134  MESSAGE = "Retrieve all V8 releases."
135
136  def ExceedsMax(self, releases):
137    return (self._options.max_releases > 0
138            and len(releases) > self._options.max_releases)
139
140  def GetMasterHashFromPush(self, title):
141    return MatchSafe(PUSH_MSG_GIT_RE.match(title))
142
143  def GetMergedPatches(self, body):
144    patches = MatchSafe(MERGE_MESSAGE_RE.search(body))
145    if not patches:
146      patches = MatchSafe(ROLLBACK_MESSAGE_RE.search(body))
147      if patches:
148        # Indicate reverted patches with a "-".
149        patches = "-%s" % patches
150    return patches
151
152  def GetMergedPatchesGit(self, body):
153    patches = []
154    for line in body.splitlines():
155      patch = MatchSafe(MERGE_MESSAGE_GIT_RE.match(line))
156      if patch:
157        patches.append(patch)
158      patch = MatchSafe(ROLLBACK_MESSAGE_GIT_RE.match(line))
159      if patch:
160        patches.append("-%s" % patch)
161    return ", ".join(patches)
162
163
164  def GetReleaseDict(
165      self, git_hash, master_position, master_hash, branch, version,
166      patches, cl_body):
167    revision = self.GetCommitPositionNumber(git_hash)
168    return {
169      # The cr commit position number on the branch.
170      "revision": revision,
171      # The git revision on the branch.
172      "revision_git": git_hash,
173      # The cr commit position number on master.
174      "master_position": master_position,
175      # The same for git.
176      "master_hash": master_hash,
177      # The branch name.
178      "branch": branch,
179      # The version for displaying in the form 3.26.3 or 3.26.3.12.
180      "version": version,
181      # The date of the commit.
182      "date": self.GitLog(n=1, format="%ci", git_hash=git_hash),
183      # Merged patches if available in the form 'r1234, r2345'.
184      "patches_merged": patches,
185      # Default for easier output formatting.
186      "chromium_revision": "",
187      # Default for easier output formatting.
188      "chromium_branch": "",
189      # Link to the CL on code review. Candiates pushes are not uploaded,
190      # so this field will be populated below with the recent roll CL link.
191      "review_link": MatchSafe(REVIEW_LINK_RE.search(cl_body)),
192      # Link to the commit message on google code.
193      "revision_link": ("https://code.google.com/p/v8/source/detail?r=%s"
194                        % revision),
195    }
196
197  def GetRelease(self, git_hash, branch):
198    self.ReadAndPersistVersion()
199    base_version = [self["major"], self["minor"], self["build"]]
200    version = ".".join(base_version)
201    body = self.GitLog(n=1, format="%B", git_hash=git_hash)
202
203    patches = ""
204    if self["patch"] != "0":
205      version += ".%s" % self["patch"]
206      if CHERRY_PICK_TITLE_GIT_RE.match(body.splitlines()[0]):
207        patches = self.GetMergedPatchesGit(body)
208      else:
209        patches = self.GetMergedPatches(body)
210
211    if SortingKey("4.2.69") <= SortingKey(version):
212      master_hash = self.GetLatestReleaseBase(version=version)
213    else:
214      # Legacy: Before version 4.2.69, the master revision was determined
215      # by commit message.
216      title = self.GitLog(n=1, format="%s", git_hash=git_hash)
217      master_hash = self.GetMasterHashFromPush(title)
218    master_position = ""
219    if master_hash:
220      master_position = self.GetCommitPositionNumber(master_hash)
221    return self.GetReleaseDict(
222        git_hash, master_position, master_hash, branch, version,
223        patches, body), self["patch"]
224
225  def GetReleasesFromBranch(self, branch):
226    self.GitReset(self.vc.RemoteBranch(branch))
227    if branch == self.vc.MasterBranch():
228      return self.GetReleasesFromMaster()
229
230    releases = []
231    try:
232      for git_hash in self.GitLog(format="%H").splitlines():
233        if VERSION_FILE not in self.GitChangedFiles(git_hash):
234          continue
235        if self.ExceedsMax(releases):
236          break  # pragma: no cover
237        if not self.GitCheckoutFileSafe(VERSION_FILE, git_hash):
238          break  # pragma: no cover
239
240        release, patch_level = self.GetRelease(git_hash, branch)
241        releases.append(release)
242
243        # Follow branches only until their creation point.
244        # TODO(machenbach): This omits patches if the version file wasn't
245        # manipulated correctly. Find a better way to detect the point where
246        # the parent of the branch head leads to the trunk branch.
247        if branch != self.vc.CandidateBranch() and patch_level == "0":
248          break
249
250    # Allow Ctrl-C interrupt.
251    except (KeyboardInterrupt, SystemExit):  # pragma: no cover
252      pass
253
254    # Clean up checked-out version file.
255    self.GitCheckoutFileSafe(VERSION_FILE, "HEAD")
256    return releases
257
258  def GetReleaseFromRevision(self, revision):
259    releases = []
260    try:
261      if (VERSION_FILE not in self.GitChangedFiles(revision) or
262          not self.GitCheckoutFileSafe(VERSION_FILE, revision)):
263        print "Skipping revision %s" % revision
264        return []  # pragma: no cover
265
266      branches = map(
267          str.strip,
268          self.Git("branch -r --contains %s" % revision).strip().splitlines(),
269      )
270      branch = ""
271      for b in branches:
272        if b.startswith("origin/"):
273          branch = b.split("origin/")[1]
274          break
275        if b.startswith("branch-heads/"):
276          branch = b.split("branch-heads/")[1]
277          break
278      else:
279        print "Could not determine branch for %s" % revision
280
281      release, _ = self.GetRelease(revision, branch)
282      releases.append(release)
283
284    # Allow Ctrl-C interrupt.
285    except (KeyboardInterrupt, SystemExit):  # pragma: no cover
286      pass
287
288    # Clean up checked-out version file.
289    self.GitCheckoutFileSafe(VERSION_FILE, "HEAD")
290    return releases
291
292
293  def RunStep(self):
294    self.GitCreateBranch(self._config["BRANCHNAME"])
295    releases = []
296    if self._options.branch == 'recent':
297      # List every release from the last 7 days.
298      revisions = self.GetRecentReleases(max_age=7 * DAY_IN_SECONDS)
299      for revision in revisions:
300        releases += self.GetReleaseFromRevision(revision)
301    elif self._options.branch == 'all':  # pragma: no cover
302      # Retrieve the full release history.
303      for branch in self.vc.GetBranches():
304        releases += self.GetReleasesFromBranch(branch)
305      releases += self.GetReleasesFromBranch(self.vc.CandidateBranch())
306      releases += self.GetReleasesFromBranch(self.vc.MasterBranch())
307    else:  # pragma: no cover
308      # Retrieve history for a specified branch.
309      assert self._options.branch in (self.vc.GetBranches() +
310          [self.vc.CandidateBranch(), self.vc.MasterBranch()])
311      releases += self.GetReleasesFromBranch(self._options.branch)
312
313    self["releases"] = sorted(releases,
314                              key=lambda r: SortingKey(r["version"]),
315                              reverse=True)
316
317
318class UpdateChromiumCheckout(Step):
319  MESSAGE = "Update the chromium checkout."
320
321  def RunStep(self):
322    cwd = self._options.chromium
323    self.GitFetchOrigin("+refs/heads/*:refs/remotes/origin/*",
324                        "+refs/branch-heads/*:refs/remotes/branch-heads/*",
325                        cwd=cwd)
326    # Update v8 checkout in chromium.
327    self.GitFetchOrigin(cwd=os.path.join(cwd, "v8"))
328
329
330def ConvertToCommitNumber(step, revision):
331  # Simple check for git hashes.
332  if revision.isdigit() and len(revision) < 8:
333    return revision
334  return step.GetCommitPositionNumber(
335      revision, cwd=os.path.join(step._options.chromium, "v8"))
336
337
338class RetrieveChromiumV8Releases(Step):
339  MESSAGE = "Retrieve V8 releases from Chromium DEPS."
340
341  def RunStep(self):
342    cwd = self._options.chromium
343
344    # All v8 revisions we are interested in.
345    releases_dict = dict((r["revision_git"], r) for r in self["releases"])
346
347    cr_releases = []
348    count_past_last_v8 = 0
349    try:
350      for git_hash in self.GitLog(
351          format="%H", grep="V8", branch="origin/master",
352          path="DEPS", cwd=cwd).splitlines():
353        deps = self.GitShowFile(git_hash, "DEPS", cwd=cwd)
354        match = DEPS_RE.search(deps)
355        if match:
356          cr_rev = self.GetCommitPositionNumber(git_hash, cwd=cwd)
357          if cr_rev:
358            v8_hsh = match.group(1)
359            cr_releases.append([cr_rev, v8_hsh])
360
361          if count_past_last_v8:
362            count_past_last_v8 += 1  # pragma: no cover
363
364          if count_past_last_v8 > 20:
365            break  # pragma: no cover
366
367          # Stop as soon as we find a v8 revision that we didn't fetch in the
368          # v8-revision-retrieval part above (i.e. a revision that's too old).
369          # Just iterate a few more times in case there were reverts.
370          if v8_hsh not in releases_dict:
371            count_past_last_v8 += 1  # pragma: no cover
372
373    # Allow Ctrl-C interrupt.
374    except (KeyboardInterrupt, SystemExit):  # pragma: no cover
375      pass
376
377    # Add the chromium ranges to the v8 candidates and master releases.
378    all_ranges = BuildRevisionRanges(cr_releases)
379
380    for hsh, ranges in all_ranges.iteritems():
381      releases_dict.get(hsh, {})["chromium_revision"] = ranges
382
383
384# TODO(machenbach): Unify common code with method above.
385class RetrieveChromiumBranches(Step):
386  MESSAGE = "Retrieve Chromium branch information."
387
388  def RunStep(self):
389    cwd = self._options.chromium
390
391    # All v8 revisions we are interested in.
392    releases_dict = dict((r["revision_git"], r) for r in self["releases"])
393
394    # Filter out irrelevant branches.
395    branches = filter(lambda r: re.match(r"branch-heads/\d+", r),
396                      self.GitRemotes(cwd=cwd))
397
398    # Transform into pure branch numbers.
399    branches = map(lambda r: int(re.match(r"branch-heads/(\d+)", r).group(1)),
400                   branches)
401
402    branches = sorted(branches, reverse=True)
403
404    cr_branches = []
405    count_past_last_v8 = 0
406    try:
407      for branch in branches:
408        deps = self.GitShowFile(
409            "refs/branch-heads/%d" % branch, "DEPS", cwd=cwd)
410        match = DEPS_RE.search(deps)
411        if match:
412          v8_hsh = match.group(1)
413          cr_branches.append([str(branch), v8_hsh])
414
415          if count_past_last_v8:
416            count_past_last_v8 += 1  # pragma: no cover
417
418          if count_past_last_v8 > 20:
419            break  # pragma: no cover
420
421          # Stop as soon as we find a v8 revision that we didn't fetch in the
422          # v8-revision-retrieval part above (i.e. a revision that's too old).
423          # Just iterate a few more times in case there were reverts.
424          if v8_hsh not in releases_dict:
425            count_past_last_v8 += 1  # pragma: no cover
426
427    # Allow Ctrl-C interrupt.
428    except (KeyboardInterrupt, SystemExit):  # pragma: no cover
429      pass
430
431    # Add the chromium branches to the v8 candidate releases.
432    all_ranges = BuildRevisionRanges(cr_branches)
433    for revision, ranges in all_ranges.iteritems():
434      releases_dict.get(revision, {})["chromium_branch"] = ranges
435
436
437class RetrieveInformationOnChromeReleases(Step):
438  MESSAGE = 'Retrieves relevant information on the latest Chrome releases'
439
440  def Run(self):
441
442    params = None
443    result_raw = self.ReadURL(
444                             OMAHA_PROXY_URL + "all.json",
445                             params,
446                             wait_plan=[5, 20]
447                             )
448    recent_releases = json.loads(result_raw)
449
450    canaries = []
451
452    for current_os in recent_releases:
453      for current_version in current_os["versions"]:
454        if current_version["channel"] != "canary":
455          continue
456
457        current_candidate = self._CreateCandidate(current_version)
458        canaries.append(current_candidate)
459
460    chrome_releases = {"canaries": canaries}
461    self["chrome_releases"] = chrome_releases
462
463  def _GetGitHashForV8Version(self, v8_version):
464    if v8_version == "N/A":
465      return ""
466
467    real_v8_version = v8_version
468    if v8_version.split(".")[3]== "0":
469      real_v8_version = v8_version[:-2]
470
471    try:
472      return self.GitGetHashOfTag(real_v8_version)
473    except GitFailedException:
474      return ""
475
476  def _CreateCandidate(self, current_version):
477    params = None
478    url_to_call = (OMAHA_PROXY_URL + "v8.json?version="
479                   + current_version["previous_version"])
480    result_raw = self.ReadURL(
481                         url_to_call,
482                         params,
483                         wait_plan=[5, 20]
484                         )
485    previous_v8_version = json.loads(result_raw)["v8_version"]
486    v8_previous_version_hash = self._GetGitHashForV8Version(previous_v8_version)
487
488    current_v8_version = current_version["v8_version"]
489    v8_version_hash = self._GetGitHashForV8Version(current_v8_version)
490
491    current_candidate = {
492                        "chrome_version": current_version["version"],
493                        "os": current_version["os"],
494                        "release_date": current_version["current_reldate"],
495                        "v8_version": current_v8_version,
496                        "v8_version_hash": v8_version_hash,
497                        "v8_previous_version": previous_v8_version,
498                        "v8_previous_version_hash": v8_previous_version_hash,
499                       }
500    return current_candidate
501
502
503class CleanUp(Step):
504  MESSAGE = "Clean up."
505
506  def RunStep(self):
507    self.CommonCleanup()
508
509
510class WriteOutput(Step):
511  MESSAGE = "Print output."
512
513  def Run(self):
514
515    output = {
516              "releases": self["releases"],
517              "chrome_releases": self["chrome_releases"],
518              }
519
520    if self._options.csv:
521      with open(self._options.csv, "w") as f:
522        writer = csv.DictWriter(f,
523                                ["version", "branch", "revision",
524                                 "chromium_revision", "patches_merged"],
525                                restval="",
526                                extrasaction="ignore")
527        for release in self["releases"]:
528          writer.writerow(release)
529    if self._options.json:
530      with open(self._options.json, "w") as f:
531        f.write(json.dumps(output))
532    if not self._options.csv and not self._options.json:
533      print output  # pragma: no cover
534
535
536class Releases(ScriptsBase):
537  def _PrepareOptions(self, parser):
538    parser.add_argument("-b", "--branch", default="recent",
539                        help=("The branch to analyze. If 'all' is specified, "
540                              "analyze all branches. If 'recent' (default) "
541                              "is specified, track beta, stable and "
542                              "candidates."))
543    parser.add_argument("-c", "--chromium",
544                        help=("The path to your Chromium src/ "
545                              "directory to automate the V8 roll."))
546    parser.add_argument("--csv", help="Path to a CSV file for export.")
547    parser.add_argument("-m", "--max-releases", type=int, default=0,
548                        help="The maximum number of releases to track.")
549    parser.add_argument("--json", help="Path to a JSON file for export.")
550
551  def _ProcessOptions(self, options):  # pragma: no cover
552    options.force_readline_defaults = True
553    return True
554
555  def _Config(self):
556    return {
557      "BRANCHNAME": "retrieve-v8-releases",
558      "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile",
559    }
560
561  def _Steps(self):
562
563    return [
564      Preparation,
565      RetrieveV8Releases,
566      UpdateChromiumCheckout,
567      RetrieveChromiumV8Releases,
568      RetrieveChromiumBranches,
569      RetrieveInformationOnChromeReleases,
570      CleanUp,
571      WriteOutput,
572    ]
573
574
575if __name__ == "__main__":  # pragma: no cover
576  sys.exit(Releases().Run())
577