1#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Updates the Chrome reference builds.
8
9Usage:
10  $ /path/to/update_reference_build.py
11  $ git commit -a
12  $ git cl upload
13"""
14
15import collections
16import logging
17import os
18import shutil
19import subprocess
20import sys
21import urllib2
22import zipfile
23
24sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils'))
25
26from py_utils import cloud_storage
27from dependency_manager import base_config
28
29
30def BuildNotFoundError(error_string):
31  raise ValueError(error_string)
32
33
34_CHROME_BINARIES_CONFIG = os.path.join(
35    os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common',
36    'py_utils', 'py_utils', 'chrome_binaries.json')
37
38CHROME_GS_BUCKET = 'chrome-unsigned'
39
40
41# Remove a platform name from this list to disable updating it.
42# Add one to enable updating it. (Must also update _PLATFORM_MAP.)
43_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64',
44                        'android_k_armeabi-v7a', 'android_l_arm64-v8a',
45                        'android_l_armeabi-v7a']
46
47# Remove a channal name from this list to disable updating it.
48# Add one to enable updating it.
49_CHANNELS_TO_UPDATE = ['stable'] # 'canary', 'dev'
50
51
52# Omaha is Chrome's autoupdate server. It reports the current versions used
53# by each platform on each channel.
54_OMAHA_PLATFORMS = { 'stable':  ['mac', 'linux', 'win', 'android'],
55                    'dev':  ['linux'], 'canary': ['mac', 'win']}
56
57
58# All of the information we need to update each platform.
59#   omaha: name omaha uses for the plaftorms.
60#   zip_name: name of the zip file to be retrieved from cloud storage.
61#   gs_build: name of the Chrome build platform used in cloud storage.
62#   destination: Name of the folder to download the reference build to.
63UpdateInfo = collections.namedtuple('UpdateInfo',
64    'omaha, gs_folder, gs_build, zip_name')
65_PLATFORM_MAP = { 'mac_x86_64': UpdateInfo(omaha='mac',
66                                           gs_folder='desktop-*',
67                                           gs_build='mac64',
68                                           zip_name='chrome-mac.zip'),
69                  'win_x86': UpdateInfo(omaha='win',
70                                        gs_folder='desktop-*',
71                                        gs_build='win',
72                                        zip_name='chrome-win.zip'),
73                  'win_AMD64': UpdateInfo(omaha='win',
74                                          gs_folder='desktop-*',
75                                          gs_build='win64',
76                                          zip_name='chrome-win64.zip'),
77                  'linux_x86_64': UpdateInfo(omaha='linux',
78                                             gs_folder='desktop-*',
79                                             gs_build='linux64',
80                                             zip_name='chrome-linux64.zip'),
81                  'android_k_armeabi-v7a': UpdateInfo(omaha='android',
82                                                      gs_folder='android-*',
83                                                      gs_build='arm',
84                                                      zip_name='Chrome.apk'),
85                  'android_l_arm64-v8a': UpdateInfo(omaha='android',
86                                                    gs_folder='android-*',
87                                                    gs_build='arm_64',
88                                                    zip_name='ChromeModern.apk'),
89                  'android_l_armeabi-v7a': UpdateInfo(omaha='android',
90                                                      gs_folder='android-*',
91                                                      gs_build='arm',
92                                                      zip_name='Chrome.apk'),
93}
94
95
96def _ChannelVersionsMap(channel):
97  rows = _OmahaReportVersionInfo(channel)
98  omaha_versions_map = _OmahaVersionsMap(rows, channel)
99  channel_versions_map = {}
100  for platform in _PLATFORMS_TO_UPDATE:
101    omaha_platform = _PLATFORM_MAP[platform].omaha
102    if omaha_platform in omaha_versions_map:
103      channel_versions_map[platform] = omaha_versions_map[omaha_platform]
104  return channel_versions_map
105
106
107def _OmahaReportVersionInfo(channel):
108  url ='https://omahaproxy.appspot.com/all?channel=%s' % channel
109  lines = urllib2.urlopen(url).readlines()
110  return [l.split(',') for l in lines]
111
112
113def _OmahaVersionsMap(rows, channel):
114  platforms = _OMAHA_PLATFORMS.get(channel, [])
115  if (len(rows) < 1 or
116      not rows[0][0:3] == ['os', 'channel', 'current_version']):
117    raise ValueError(
118        'Omaha report is not in the expected form: %s.' % rows)
119  versions_map = {}
120  for row in rows[1:]:
121    if row[1] != channel:
122      raise ValueError(
123          'Omaha report contains a line with the channel %s' % row[1])
124    if row[0] in platforms:
125      versions_map[row[0]] = row[2]
126  logging.warn('versions map: %s' % versions_map)
127  if not all(platform in versions_map for platform in platforms):
128    raise ValueError(
129        'Omaha report did not contain all desired platforms for channel %s' % channel)
130  return versions_map
131
132
133def _QueuePlatformUpdate(platform, version, config, channel):
134  """ platform: the name of the platform for the browser to
135      be downloaded & updated from cloud storage. """
136  platform_info = _PLATFORM_MAP[platform]
137  filename = platform_info.zip_name
138  # remote_path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip
139  remote_path = '%s/%s/%s/%s' % (
140      platform_info.gs_folder, version, platform_info.gs_build, filename)
141  if not cloud_storage.Exists(CHROME_GS_BUCKET, remote_path):
142    raise BuildNotFoundError(
143        'Failed to find %s build for version %s at path %s.' % (platform, version, remote_path))
144  reference_builds_folder = os.path.join(
145      os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build',
146      'reference_builds', channel)
147  if not os.path.exists(reference_builds_folder):
148    os.makedirs(reference_builds_folder)
149  local_dest_path = os.path.join(reference_builds_folder, filename)
150  cloud_storage.Get(CHROME_GS_BUCKET, remote_path, local_dest_path)
151  config.AddCloudStorageDependencyUpdateJob(
152      'chrome_%s' % channel, platform, local_dest_path, version=version,
153      execute_job=False)
154
155
156def UpdateBuilds():
157  config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True)
158  for channel in _CHANNELS_TO_UPDATE:
159    channel_versions_map = _ChannelVersionsMap(channel)
160    for platform in channel_versions_map:
161      print 'Downloading Chrome (%s channel) on %s' % (channel, platform)
162      current_version = config.GetVersion('chrome_%s' % channel, platform)
163      channel_version =  channel_versions_map.get(platform)
164      print 'current: %s, channel: %s' % (current_version, channel_version)
165      if current_version and current_version == channel_version:
166        continue
167      _QueuePlatformUpdate(platform, channel_version, config, channel)
168    # TODO: move execute update jobs here, and add committing/uploading the cl.
169
170  print 'Updating chrome builds with downloaded binaries'
171  config.ExecuteUpdateJobs(force=True)
172
173
174def main():
175  logging.getLogger().setLevel(logging.DEBUG)
176  #TODO(aiolos): alert sheriffs via email when an error is seen.
177  #This should be added when alerts are added when updating the build.
178  UpdateBuilds()
179  # TODO(aiolos): Add --commit flag. crbug.com/547229
180
181if __name__ == '__main__':
182  main()
183