1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2020 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Checks for various upstream events with the Rust toolchain.
8
9Sends an email if something interesting (probably) happened.
10"""
11
12# pylint: disable=cros-logging-import
13
14import argparse
15import itertools
16import json
17import logging
18import pathlib
19import re
20import shutil
21import subprocess
22import sys
23import time
24from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple
25
26from cros_utils import email_sender
27from cros_utils import tiny_render
28
29
30def gentoo_sha_to_link(sha: str) -> str:
31  """Gets a URL to a webpage that shows the Gentoo commit at `sha`."""
32  return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}'
33
34
35def send_email(subject: str, body: List[tiny_render.Piece]) -> None:
36  """Sends an email with the given title and body to... whoever cares."""
37  email_sender.EmailSender().SendX20Email(
38      subject=subject,
39      identifier='rust-watch',
40      well_known_recipients=['cros-team'],
41      text_body=tiny_render.render_text_pieces(body),
42      html_body=tiny_render.render_html_pieces(body),
43  )
44
45
46class RustReleaseVersion(NamedTuple):
47  """Represents a version of Rust's stable compiler."""
48  major: int
49  minor: int
50  patch: int
51
52  @staticmethod
53  def from_string(version_string: str) -> 'RustReleaseVersion':
54    m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string)
55    if not m:
56      raise ValueError(f"{version_string!r} isn't a valid version string")
57    return RustReleaseVersion(*[int(x) for x in m.groups()])
58
59  def __str__(self) -> str:
60    return f'{self.major}.{self.minor}.{self.patch}'
61
62  def to_json(self) -> str:
63    return str(self)
64
65  @staticmethod
66  def from_json(s: str) -> 'RustReleaseVersion':
67    return RustReleaseVersion.from_string(s)
68
69
70class State(NamedTuple):
71  """State that we keep around from run to run."""
72  # The last Rust release tag that we've seen.
73  last_seen_release: RustReleaseVersion
74
75  # We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen
76  # that updates it.
77  last_gentoo_sha: str
78
79  def to_json(self) -> Dict[str, Any]:
80    return {
81        'last_seen_release': self.last_seen_release.to_json(),
82        'last_gentoo_sha': self.last_gentoo_sha,
83    }
84
85  @staticmethod
86  def from_json(s: Dict[str, Any]) -> 'State':
87    return State(
88        last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']),
89        last_gentoo_sha=s['last_gentoo_sha'],
90    )
91
92
93def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]:
94  """Parses `git ls-remote --tags` output into Rust stable release versions."""
95  refs_tags = 'refs/tags/'
96  for line in lines:
97    _sha, tag = line.split(None, 1)
98    tag = tag.strip()
99    # Each tag has an associated 'refs/tags/name^{}', which is the actual
100    # object that the tag points to. That's irrelevant to us.
101    if tag.endswith('^{}'):
102      continue
103
104    if not tag.startswith(refs_tags):
105      continue
106
107    short_tag = tag[len(refs_tags):]
108    # There are a few old versioning schemes. Ignore them.
109    if short_tag.startswith('0.') or short_tag.startswith('release-'):
110      continue
111    yield RustReleaseVersion.from_string(short_tag)
112
113
114def fetch_most_recent_release() -> RustReleaseVersion:
115  """Fetches the most recent stable `rustc` version."""
116  result = subprocess.run(
117      ['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'],
118      check=True,
119      stdin=None,
120      capture_output=True,
121      encoding='utf-8',
122  )
123  tag_lines = result.stdout.strip().splitlines()
124  return max(parse_release_tags(tag_lines))
125
126
127class GitCommit(NamedTuple):
128  """Represents a single git commit."""
129  sha: str
130  subject: str
131
132
133def update_git_repo(git_dir: pathlib.Path) -> None:
134  """Updates the repo at `git_dir`, retrying a few times on failure."""
135  for i in itertools.count(start=1):
136    result = subprocess.run(
137        ['git', 'fetch', 'origin'],
138        check=False,
139        cwd=str(git_dir),
140        stdin=None,
141    )
142
143    if not result.returncode:
144      break
145
146    if i == 5:
147      # 5 attempts is too many. Something else may be wrong.
148      result.check_returncode()
149
150    sleep_time = 60 * i
151    logging.error("Failed updating gentoo's repo; will try again in %ds...",
152                  sleep_time)
153    time.sleep(sleep_time)
154
155
156def get_new_gentoo_commits(git_dir: pathlib.Path,
157                           most_recent_sha: str) -> List[GitCommit]:
158  """Gets commits to dev-lang/rust since `most_recent_sha`.
159
160  Older commits come earlier in the returned list.
161  """
162  commits = subprocess.run(
163      [
164          'git',
165          'log',
166          '--format=%H %s',
167          f'{most_recent_sha}..origin/master',
168          '--',
169          'dev-lang/rust',
170      ],
171      capture_output=True,
172      check=False,
173      cwd=str(git_dir),
174      encoding='utf-8',
175  )
176
177  if commits.returncode:
178    logging.error('Error getting new gentoo commits; stderr:\n%s',
179                  commits.stderr)
180    commits.check_returncode()
181
182  results = []
183  for line in commits.stdout.strip().splitlines():
184    sha, subject = line.strip().split(None, 1)
185    results.append(GitCommit(sha=sha, subject=subject))
186
187  # `git log` outputs things in newest -> oldest order.
188  results.reverse()
189  return results
190
191
192def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str:
193  """Sets up a gentoo git repo at the given directory. Returns HEAD."""
194  subprocess.run(
195      [
196          'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git',
197          str(git_dir)
198      ],
199      stdin=None,
200      check=True,
201  )
202
203  head_rev = subprocess.run(
204      ['git', 'rev-parse', 'HEAD'],
205      cwd=str(git_dir),
206      check=True,
207      stdin=None,
208      capture_output=True,
209      encoding='utf-8',
210  )
211  return head_rev.stdout.strip()
212
213
214def read_state(state_file: pathlib.Path) -> State:
215  """Reads state from the given file."""
216  with state_file.open(encoding='utf-8') as f:
217    return State.from_json(json.load(f))
218
219
220def atomically_write_state(state_file: pathlib.Path, state: State) -> None:
221  """Writes state to the given file."""
222  temp_file = pathlib.Path(str(state_file) + '.new')
223  with temp_file.open('w', encoding='utf-8') as f:
224    json.dump(state.to_json(), f)
225  temp_file.rename(state_file)
226
227
228def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion,
229                        new_gentoo_commits: List[GitCommit]
230                       ) -> Optional[Tuple[str, List[tiny_render.Piece]]]:
231  """Creates an email given our new state, if doing so is appropriate."""
232  subject_pieces = []
233  body_pieces = []
234
235  if newest_release > old_state.last_seen_release:
236    subject_pieces.append('new rustc release detected')
237    body_pieces.append(f'Rustc tag for v{newest_release} was found.')
238
239  if new_gentoo_commits:
240    # Separate the sections a bit for prettier output.
241    if body_pieces:
242      body_pieces += [tiny_render.line_break, tiny_render.line_break]
243
244    if len(new_gentoo_commits) == 1:
245      subject_pieces.append('new rust ebuild commit detected')
246      body_pieces.append('commit:')
247    else:
248      subject_pieces.append('new rust ebuild commits detected')
249      body_pieces.append('commits (newest first):')
250
251    commit_lines = []
252    for commit in new_gentoo_commits:
253      commit_lines.append([
254          tiny_render.Link(
255              gentoo_sha_to_link(commit.sha),
256              commit.sha[:12],
257          ),
258          f': {commit.subject}',
259      ])
260
261    body_pieces.append(tiny_render.UnorderedList(commit_lines))
262
263  if not subject_pieces:
264    return None
265
266  subject = '[rust-watch] ' + '; '.join(subject_pieces)
267  return subject, body_pieces
268
269
270def main(argv: List[str]) -> None:
271  logging.basicConfig(level=logging.INFO)
272
273  parser = argparse.ArgumentParser(
274      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
275  parser.add_argument(
276      '--state_dir', required=True, help='Directory to store state in.')
277  parser.add_argument(
278      '--skip_email', action='store_true', help="Don't send an email.")
279  parser.add_argument(
280      '--skip_state_update',
281      action='store_true',
282      help="Don't update the state file. Doesn't apply to initial setup.")
283  opts = parser.parse_args(argv)
284
285  state_dir = pathlib.Path(opts.state_dir)
286  state_file = state_dir / 'state.json'
287  gentoo_subdir = state_dir / 'upstream-gentoo'
288  if not state_file.exists():
289    logging.info("state_dir isn't fully set up; doing that now.")
290
291    # Could be in a partially set-up state.
292    if state_dir.exists():
293      logging.info('incomplete state_dir detected; removing.')
294      shutil.rmtree(str(state_dir))
295
296    state_dir.mkdir(parents=True)
297    most_recent_release = fetch_most_recent_release()
298    most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir)
299    atomically_write_state(
300        state_file,
301        State(
302            last_seen_release=most_recent_release,
303            last_gentoo_sha=most_recent_gentoo_commit,
304        ),
305    )
306    # Running through this _should_ be a nop, but do it anyway. Should make any
307    # bugs more obvious on the first run of the script.
308
309  prior_state = read_state(state_file)
310  logging.info('Last state was %r', prior_state)
311
312  most_recent_release = fetch_most_recent_release()
313  logging.info('Most recent Rust release is %s', most_recent_release)
314
315  logging.info('Fetching new commits from Gentoo')
316  update_git_repo(gentoo_subdir)
317  new_commits = get_new_gentoo_commits(gentoo_subdir,
318                                       prior_state.last_gentoo_sha)
319  logging.info('New commits: %r', new_commits)
320
321  maybe_email = maybe_compose_email(prior_state, most_recent_release,
322                                    new_commits)
323
324  if maybe_email is None:
325    logging.info('No updates to send')
326  else:
327    title, body = maybe_email
328    if opts.skip_email:
329      logging.info('Skipping sending email with title %r and contents\n%s',
330                   title, tiny_render.render_html_pieces(body))
331    else:
332      logging.info('Sending email')
333      send_email(title, body)
334
335  if opts.skip_state_update:
336    logging.info('Skipping state update, as requested')
337    return
338
339  newest_sha = (
340      new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha)
341  atomically_write_state(
342      state_file,
343      State(
344          last_seen_release=most_recent_release,
345          last_gentoo_sha=newest_sha,
346      ),
347  )
348
349
350if __name__ == '__main__':
351  sys.exit(main(sys.argv[1:]))
352