1#!/usr/bin/env python3
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import argparse
17import hashlib
18import logging
19import os
20import shutil
21import subprocess
22import stat
23import sys
24import tempfile
25import zipfile
26
27from collections import namedtuple
28from platform import system, machine
29
30# The format for the deps below is the following:
31# (target_folder, source_url, sha1, target_platform)
32# |source_url| can be either a git repo or a http url.
33# If a git repo, |checksum| is the SHA1 committish that will be checked out.
34# If a http url, |checksum| is the SHA256 of the downloaded file.
35# If the url is a .zip or .tgz file it will be automatically deflated under
36# |target_folder|, taking care of stripping the root folder if it's a single
37# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
38# instead just buildtools/protobuf).
39# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
40# |target_arch| is either 'x64', 'aarch64' or 'all'
41# in both cases the dep only applies on matching platforms
42# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
43# case of MacOS universal binaries.
44Dependency = namedtuple(
45    'Dependency',
46    ['target_folder', 'source_url', 'checksum', 'target_os', 'target_arch'])
47
48# This is to remove old directories when build tools get {re,}moved. This is to
49# avoid accidentally referring to stale dir in custom user scripts.
50CLEANUP_OLD_DIRS = [
51    'buildtools/nodejs',  # Moved to buildtools/{mac,linux64}/nodejs
52    'buildtools/emsdk',  # Moved to buildtools/{mac,linux64}/emsdk
53    'buildtools/test_data',  # Moved to test/data by r.android.com/1539381 .
54    'buildtools/d8',  # Removed by r.android.com/1424334 .
55]
56
57# Dependencies required to build code on the host or when targeting desktop OS.
58BUILD_DEPS_TOOLCHAIN_HOST = [
59    # GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
60    # git_revision:83dad00afb232d7235dd70dff1ee90292d72a01e .
61    Dependency(
62        'buildtools/mac/gn',
63        'https://storage.googleapis.com/perfetto/gn-mac-1695-83dad00a',
64        '513d3adeb56b745e62af4e3ccb76b76f023c6aaa25d6a2be9a89e44cd10a4c1a',
65        'darwin', 'x64'),
66    Dependency(
67        'buildtools/linux64/gn',
68        'https://storage.googleapis.com/perfetto/gn-linux64-1695-83dad00a',
69        '4f589364153f182b05cd845e93407489d6ce8acc03290c897928a7bd22b20cce',
70        'linux', 'x64'),
71    Dependency(
72        'buildtools/win/gn.exe',
73        'https://storage.googleapis.com/perfetto/gn-win-1695-83dad00a',
74        '908c29556539292203d2952ebf55df03697cbc7cf526a3e295f31ba2576e4cac',
75        'windows', 'x64'),
76
77    # clang-format
78    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.sha1
79    Dependency(
80        'buildtools/mac/clang-format',
81        'https://storage.googleapis.com/chromium-clang-format/62bde1baa7196ad9df969fc1f06b66360b1a927b',
82        '6df686a937443cbe6efc013467a7ba5f98d3f187eb7765bb7abc6ce47626cf66',
83        'darwin', 'x64'),
84    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
85    Dependency(
86        'buildtools/linux64/clang-format',
87        'https://storage.googleapis.com/chromium-clang-format/1baf0089e895c989a311b6a38ed94d0e8be4c0a7',
88        'd02a97a87e8c28898033aaf5986967b24dc47ebd5b376e1cd93e5009f22cd75e',
89        'linux', 'x64'),
90    # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
91    Dependency(
92        'buildtools/win/clang-format.exe',
93        'https://storage.googleapis.com/chromium-clang-format/d4afd4eba27022f5f6d518133aebde57281677c9',
94        '2ba1b4d3ade90ea80316890b598ab5fc16777572be26afec6ce23117da121b80',
95        'windows', 'x64'),
96
97    # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
98    Dependency(
99        'buildtools/clang_format/script',
100        'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git',
101        '96636aa0e9f047f17447f2d45a094d0b59ed7917', 'all', 'all'),
102
103    # Ninja
104    Dependency(
105        'buildtools/mac/ninja',
106        'https://storage.googleapis.com/perfetto/ninja-mac-c15b0698da038b2bd2e8970c14c75fadc06b1add',
107        '4224b90734590b0148ad8ee63ee7b295e88e0652e4d1f4271ef2b91d880b0e19',
108        'darwin', 'x64'),
109    Dependency(
110        'buildtools/linux64/ninja',
111        'https://storage.googleapis.com/perfetto/ninja-linux64-c866952bda50c29a669222477309287119bbb7e8',
112        '54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
113        'linux', 'x64'),
114    Dependency(
115        'buildtools/win/ninja.exe',
116        'https://storage.googleapis.com/perfetto/ninja-win-4a5f05c24afef05ef03329a1bbfedee0678b524a',
117        '6f8af488be74ed8787d04e107080d05330587a4198ba047bd5b7f5b0c3150d61',
118        'windows', 'x64'),
119
120    # Keep the revision in sync with Chrome's PACKAGE_VERSION in
121    # tools/clang/scripts/update.py.
122    Dependency(
123        'buildtools/linux64/clang.tgz',
124        'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-12-init-5035-gd0abc757-3.tgz',
125        'b0c3015209b6d624844ad230064eb5c9b4429a2eafd4854981e73217c563d93d',
126        'linux', 'x64'),
127    Dependency(
128        'buildtools/win/clang.tgz',
129        'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-12-init-5035-gd0abc757-3.tgz',
130        'b2854d871a466e3a060469b5edb24ca355ef64576d38778f64acbd3c6d7cf530',
131        'windows', 'x64'),
132]
133
134BUILD_DEPS_HOST = [
135    # Keep in sync with Android's //external/googletest/README.version.
136    Dependency(
137        'buildtools/googletest',
138        'https://android.googlesource.com/platform/external/googletest.git',
139        '3f05f651ae3621db58468153e32016bc1397800b', 'all', 'all'),
140
141    # Keep in sync with Chromium's //third_party/protobuf.
142    Dependency(
143        'buildtools/protobuf',
144        'https://chromium.googlesource.com/external/github.com/google/protobuf.git',
145        '6a59a2ad1f61d9696092f79b6d74368b4d7970a3',  # refs/tags/v3.9.0
146        'all', 'all'),
147
148    # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
149    # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
150    Dependency(
151        'buildtools/libcxx',
152        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
153        'd9040c75cfea5928c804ab7c235fed06a63f743a', 'all', 'all'),
154    Dependency(
155        'buildtools/libcxxabi',
156        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
157        '196ba1aaa8ac285d94f4ea8d9836390a45360533', 'all', 'all'),
158    Dependency(
159        'buildtools/libunwind',
160        'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
161        'd999d54f4bca789543a2eb6c995af2d9b5a1f3ed', 'all', 'all'),
162
163    # Keep in sync with chromium DEPS.
164    Dependency(
165        'buildtools/libfuzzer',
166        'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
167        'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
168
169    # Benchmarking tool.
170    Dependency(
171        'buildtools/benchmark',
172        'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
173        '090faecb454fbd6e6e17a75ef8146acb037118d4', 'all', 'all'),
174
175    # Libbacktrace, for stacktraces in Linux/Android debug builds.
176    # From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
177    Dependency(
178        'buildtools/libbacktrace.zip',
179        'https://storage.googleapis.com/perfetto/libbacktrace-177940370e4a6b2509e92a0aaa9749184e64af43.zip',
180        '21ac9a4209f7aeef766c482be53a7fa365063c031c7077e2070b491202983b31',
181        'all', 'all'),
182
183    # Sqlite for the trace processing library.
184    # This is the amalgamated source whose compiled output is meant to be faster.
185    # We still pull the full source for the extensions (which are not available
186    # in the amalgamation).
187    Dependency(
188        'buildtools/sqlite.zip',
189        'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3350400.zip',
190        'f3bf0df69f5de0675196f4644e05d07dbc698d674dc563a12eff17d5b215cdf5',
191        'all', 'all'),
192    Dependency(
193        'buildtools/sqlite_src',
194        'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
195        'ee3686eb50c0e3dbb087c9a0976f7e37e1b014ae',  # refs/tags/version-3.32.3.
196        'all', 'all'),
197
198    # JsonCpp for legacy json import. Used only by the trace processor in
199    # standalone builds.
200    Dependency(
201        'buildtools/jsoncpp',
202        'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
203        '6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2',  # refs/tags/1.9.3.
204        'all', 'all'),
205
206    # These dependencies are for libunwindstack, which is used by src/profiling.
207    Dependency('buildtools/android-core',
208               'https://android.googlesource.com/platform/system/core.git',
209               '9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
210    Dependency(
211        'buildtools/android-unwinding',
212        'https://android.googlesource.com/platform/system/unwinding.git',
213        '5150e1292ec6b16e4717e86b9e3cfb855eec18a3', 'all', 'all'),
214    Dependency('buildtools/android-logging',
215               'https://android.googlesource.com/platform/system/logging.git',
216               '7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
217    Dependency('buildtools/android-libbase',
218               'https://android.googlesource.com/platform/system/libbase.git',
219               '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
220    Dependency(
221        'buildtools/android-libprocinfo',
222        'https://android.googlesource.com/platform/system/libprocinfo.git',
223        'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
224    Dependency('buildtools/lzma',
225               'https://android.googlesource.com/platform/external/lzma.git',
226               '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
227    Dependency('buildtools/zlib',
228               'https://android.googlesource.com/platform/external/zlib.git',
229               '5c85a2da4c13eda07f69d81a1579a5afddd35f59', 'all', 'all'),
230    Dependency('buildtools/bionic',
231               'https://android.googlesource.com/platform/bionic.git',
232               '332065d57e734b65f56474d136d22d767e36cbcd', 'all', 'all'),
233
234    # Example traces for regression tests.
235    Dependency(
236        'test/data.zip',
237        'https://storage.googleapis.com/perfetto/test-data-20210513-224349.zip',
238        '3dcc146f4ce38d17fd1f8c4c65af07e7cf7c5c4cb8aa4c7bf73ec3a095d997d1',
239        'all', 'all',
240    ),
241
242    # Linenoise, used only by trace_processor in standalone builds.
243    Dependency('buildtools/linenoise',
244               'https://fuchsia.googlesource.com/third_party/linenoise.git',
245               'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
246]
247
248# Dependencies required to build Android code.
249# URLs and SHA1s taken from:
250# - https://dl.google.com/android/repository/repository-11.xml
251# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
252BUILD_DEPS_ANDROID = [
253    # Android NDK
254    Dependency(
255        'buildtools/ndk.zip',
256        'https://dl.google.com/android/repository/android-ndk-r21e-darwin-x86_64.zip',
257        '437278103a3db12632c05b1be5c41bbb8522791a67e415cc54411a65366f499d',
258        'darwin', 'x64'),
259    Dependency(
260        'buildtools/ndk.zip',
261        'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
262        'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
263        'linux', 'x64'),
264]
265
266# Dependencies required to run Android tests.
267TEST_DEPS_ANDROID = [
268    # Android emulator images.
269    Dependency(
270        'buildtools/aosp-arm.zip',
271        'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
272        'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
273        'all', 'all'),
274
275    # platform-tools.zip contains adb binaries.
276    Dependency(
277        'buildtools/android_sdk/platform-tools.zip',
278        'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
279        '98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
280        'darwin', 'x64'),
281    Dependency(
282        'buildtools/android_sdk/platform-tools.zip',
283        'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
284        '90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
285        'linux', 'x64'),
286
287    # Android emulator binaries.
288    Dependency(
289        'buildtools/emulator',
290        'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
291        '4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
292]
293
294# This variable is updated by tools/roll-catapult-trace-viewer.
295CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
296
297TYPEFACES_SHA256 = 'b3f0f14eeecd4555ae94f897ec246b2c6e046ce0ea417407553f5767e7812575'
298
299UI_DEPS = [
300    Dependency(
301        'buildtools/mac/nodejs.tgz',
302        'https://storage.googleapis.com/chromium-nodejs/14.15.4/17ba7216e09de1bffb9dc80b7ec617a1cee40330',
303        'b81a466347d2ae34b1370b6681ba173e9fb082338170a41624b37be7a2052b7e',
304        'darwin', 'x64'),
305    Dependency(
306        'buildtools/linux64/nodejs.tgz',
307        'https://storage.googleapis.com/chromium-nodejs/14.15.4/b2e40ddbac04d05baafbb007f203c6663c9d4ca9',
308        '5aa88f1e2bf036950790277f3431634f64044ec78362f3e4f0dc8da28d61e9a4',
309        'linux', 'x64'),
310    Dependency(
311        'buildtools/mac/emsdk.tgz',
312        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
313        'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
314        'darwin', 'x64'),
315    Dependency(
316        'buildtools/linux64/emsdk.tgz',
317        'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
318        'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
319        'linux', 'x64'),
320    Dependency(
321        'buildtools/catapult_trace_viewer.tgz',
322        'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
323        % CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
324    Dependency(
325        'buildtools/typefaces.tgz',
326        'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
327        TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all')
328]
329
330ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
331UI_DIR = os.path.join(ROOT_DIR, 'ui')
332TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
333NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
334
335
336def DownloadURL(url, out_file):
337  subprocess.check_call(['curl', '-L', '-#', '-o', out_file, url])
338
339
340def GetArch():
341  arch = machine()
342  if arch == 'aarch64':
343    return 'aarch64'
344  else:
345    # Assume everything else is x64 matching previous behaviour.
346    return 'x64'
347
348
349def ReadFile(path):
350  if not os.path.exists(path):
351    return None
352  with open(path) as f:
353    return f.read().strip()
354
355
356def MkdirRecursive(path):
357  # Works with both relative and absolute paths
358  cwd = '/' if path.startswith('/') else ROOT_DIR
359  for part in path.split('/'):
360    cwd = os.path.join(cwd, part)
361    if not os.path.exists(cwd):
362      os.makedirs(cwd)
363    else:
364      assert (os.path.isdir(cwd))
365
366
367def HashLocalFile(path):
368  if not os.path.exists(path):
369    return None
370  with open(path, 'rb') as f:
371    return hashlib.sha256(f.read()).hexdigest()
372
373
374def ExtractZipfilePreservePermissions(zf, info, path):
375  zf.extract(info.filename, path=path)
376  target_path = os.path.join(path, info.filename)
377  min_acls = 0o755 if info.filename.endswith('/') else 0o644
378  os.chmod(target_path, (info.external_attr >> 16) | min_acls)
379
380
381def IsGitRepoCheckoutOutAtRevision(path, revision):
382  return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
383
384
385def RmtreeIfExists(path):
386  # Git creates read-only files on windows, which cause failures with rmtree.
387  # This seems the socially accepted way to deal with it.
388  # See https://bugs.python.org/issue19643 .
389  def del_read_only_for_windows(_action, name, _exc):
390    os.chmod(name, stat.S_IWRITE)
391    os.remove(name)
392
393  if not os.path.exists(path):
394    return
395  buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
396  test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
397  if (not os.path.abspath(path).startswith(buildtools_path) and
398      not os.path.abspath(path).startswith(test_path)):
399    # Safety check to prevent that some merge confilct ends up doing some
400    # rm -rf / or similar.
401    logging.fatal(
402      'Cannot remove %s: outside of buildtools and test/data', path)
403    sys.exit(1)
404  logging.info('Removing %s' % path)
405  shutil.rmtree(path, onerror=del_read_only_for_windows)
406
407
408def CheckoutGitRepo(path, git_url, revision, check_only):
409  if IsGitRepoCheckoutOutAtRevision(path, revision):
410    return False
411  if check_only:
412    return True
413  path = path.replace('/', os.sep)
414  RmtreeIfExists(path)
415  MkdirRecursive(path)
416  logging.info('Fetching %s @ %s into %s', git_url, revision, path)
417  subprocess.check_call(['git', 'init', path], cwd=path)
418  subprocess.check_call(
419      ['git', 'fetch', '--quiet', '--depth', '1', git_url, revision], cwd=path)
420  subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
421  assert (IsGitRepoCheckoutOutAtRevision(path, revision))
422  return True
423
424
425def InstallNodeModules(force_clean=False):
426  if force_clean:
427    node_modules = os.path.join(UI_DIR, 'node_modules')
428    logging.info('Clearing %s', node_modules)
429    subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
430                          cwd=ROOT_DIR)
431  logging.info("Running npm install in {0}".format(UI_DIR))
432  subprocess.check_call(
433      [os.path.join(TOOLS_DIR, 'npm'), 'install', '--no-save'], cwd=UI_DIR)
434  # pbjs has the bad habit of installing extra packages on its first run. Run
435  # it here, so we avoid fetches while building.
436  node_bin = os.path.join(TOOLS_DIR, 'node')
437  pbjs = [node_bin, 'node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
438  subprocess.call(pbjs, cwd=UI_DIR)
439  with open(NODE_MODULES_STATUS_FILE, 'w') as f:
440    f.write(HashLocalFile(os.path.join(UI_DIR, 'package-lock.json')))
441
442
443def CheckNodeModules():
444  """Returns True if the modules are up-to-date.
445
446  There doesn't seem to be an easy way to check node modules versions. Instead
447  just check if package-lock.json changed since the last `npm install` call.
448  """
449  if not os.path.exists(NODE_MODULES_STATUS_FILE):
450    return False
451  with open(NODE_MODULES_STATUS_FILE, 'r') as f:
452    actual = f.read()
453  expected = HashLocalFile(os.path.join(UI_DIR, 'package-lock.json'))
454  return expected == actual
455
456
457def CheckHashes():
458  for deps in [BUILD_DEPS_HOST, BUILD_DEPS_ANDROID, TEST_DEPS_ANDROID, UI_DEPS]:
459    for dep in deps:
460      if dep.source_url.endswith('.git'):
461        continue
462      logging.info('Downloading %s from %s', dep.target_platform,
463                   dep.source_url)
464      with tempfile.NamedTemporaryFile(delete=False) as f:
465        f.close()
466        DownloadURL(dep.source_url, f.name)
467        actual_checksum = HashLocalFile(f.name)
468        os.unlink(f.name)
469        if (actual_checksum != dep.checksum):
470          logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
471              dep.source_url, dep.checksum, actual_checksum))
472
473
474def Main():
475  parser = argparse.ArgumentParser()
476  parser.add_argument('--android', action='store_true')
477  parser.add_argument('--ui', action='store_true')
478  parser.add_argument('--check-only')
479  parser.add_argument('--filter', default='')
480  parser.add_argument('--verify', help='Check all URLs', action='store_true')
481  parser.add_argument('--no-toolchain', help='Do not download toolchain',
482                      action='store_true')
483  args = parser.parse_args()
484  if args.verify:
485    CheckHashes()
486    return 0
487  deps = BUILD_DEPS_HOST
488  if not args.no_toolchain:
489    deps += BUILD_DEPS_TOOLCHAIN_HOST
490  if args.android:
491    deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
492  if args.ui:
493    deps += UI_DEPS
494  deps_updated = False
495  nodejs_updated = False
496
497  for old_dir in CLEANUP_OLD_DIRS:
498    RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
499
500  for dep in deps:
501    target_os = system().lower()
502    target_arch = GetArch()
503    matches_os = dep.target_os == 'all' or target_os == dep.target_os
504    matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
505    if not matches_os or not matches_arch:
506      continue
507    if args.filter and args.filter not in dep.target_folder:
508      continue
509    local_path = os.path.join(ROOT_DIR, dep.target_folder)
510    if dep.source_url.endswith('.git'):
511      deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
512                                      args.check_only)
513      continue
514    is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz')
515    zip_target_dir = local_path[:-4] if is_zip else None
516    zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None
517
518    if ((not is_zip and HashLocalFile(local_path) == dep.checksum) or
519        (is_zip and ReadFile(zip_dir_stamp) == dep.checksum)):
520      continue
521    deps_updated = True
522    if args.check_only:
523      continue
524    MkdirRecursive(os.path.dirname(dep.target_folder))
525    if HashLocalFile(local_path) != dep.checksum:
526      download_path = local_path + '.tmp'
527      logging.info('Downloading %s from %s', local_path, dep.source_url)
528      DownloadURL(dep.source_url, download_path)
529      os.chmod(download_path, 0o755)
530      actual_checksum = HashLocalFile(download_path)
531      if (actual_checksum != dep.checksum):
532        os.remove(download_path)
533        logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
534            download_path, dep.checksum, actual_checksum))
535        return 1
536      shutil.move(download_path, local_path)
537      if 'nodejs' in dep.target_folder:
538        nodejs_updated = True
539
540    assert (HashLocalFile(local_path) == dep.checksum)
541
542    if is_zip:
543      logging.info('Extracting %s into %s' % (local_path, zip_target_dir))
544      assert (os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR)
545      RmtreeIfExists(zip_target_dir)
546
547      # Decompress the archive.
548      if local_path.endswith('.tgz'):
549        MkdirRecursive(zip_target_dir)
550        subprocess.check_call(['tar', '-xf', local_path], cwd=zip_target_dir)
551      elif local_path.endswith('.zip'):
552        with zipfile.ZipFile(local_path, 'r') as zf:
553          for info in zf.infolist():
554            ExtractZipfilePreservePermissions(zf, info, zip_target_dir)
555
556      # If the zip contains one root folder, rebase one level up moving all
557      # its sub files and folders inside |target_dir|.
558      subdir = os.listdir(zip_target_dir)
559      if len(subdir) == 1:
560        subdir = os.path.join(zip_target_dir, subdir[0])
561        if os.path.isdir(subdir):
562          for subf in os.listdir(subdir):
563            shutil.move(os.path.join(subdir, subf), zip_target_dir)
564          os.rmdir(subdir)
565
566      # Create stamp and remove the archive.
567      with open(zip_dir_stamp, 'w') as stamp_file:
568        stamp_file.write(dep.checksum)
569      os.remove(local_path)
570
571  if args.ui:
572    # Needs to happen after nodejs is installed above.
573    if args.check_only:
574      deps_updated |= not CheckNodeModules()
575    else:
576      InstallNodeModules(force_clean=nodejs_updated)
577
578  if args.check_only:
579    if not deps_updated:
580      with open(args.check_only, 'w') as f:
581        f.write('OK')  # The content is irrelevant, just keep GN happy.
582      return 0
583    argz = ' '.join([x for x in sys.argv[1:] if not x.startswith('--check-only')])
584    print('\033[91mBuild deps are stale. ' +
585          'Please run tools/install-build-deps %s\033[0m' % argz)
586    return 1
587
588  if deps_updated:
589    # Stale binary files may be compiled against old sysroot headers that aren't
590    # tracked by gn.
591    logging.warning('Remember to run "gn clean <output_directory>" ' +
592                    'to avoid stale binary files.')
593
594
595if __name__ == '__main__':
596  logging.basicConfig(level=logging.INFO)
597  sys.exit(Main())
598