1#!/usr/bin/env python
2# Copyright 2014 Google Inc.
3#
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8"""Parse a DEPS file and git checkout all of the dependencies.
9
10Args:
11  An optional list of deps_os values.
12
13Environment Variables:
14  GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in
15  your default path.
16
17  GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
18  will use the file ../DEPS relative to this script's directory.
19
20  GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
21
22Git Config:
23  To disable syncing of a single repository:
24      cd path/to/repository
25      git config sync-deps.disable true
26
27  To re-enable sync:
28      cd path/to/repository
29      git config --unset sync-deps.disable
30"""
31
32
33import os
34import subprocess
35import sys
36import threading
37
38
39def git_executable():
40  """Find the git executable.
41
42  Returns:
43      A string suitable for passing to subprocess functions, or None.
44  """
45  envgit = os.environ.get('GIT_EXECUTABLE')
46  searchlist = ['git']
47  if envgit:
48    searchlist.insert(0, envgit)
49  with open(os.devnull, 'w') as devnull:
50    for git in searchlist:
51      try:
52        subprocess.call([git, '--version'], stdout=devnull)
53      except (OSError,):
54        continue
55      return git
56  return None
57
58
59DEFAULT_DEPS_PATH = os.path.normpath(
60  os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
61
62
63def usage(deps_file_path = None):
64  sys.stderr.write(
65    'Usage: run to grab dependencies, with optional platform support:\n')
66  sys.stderr.write('  %s %s' % (sys.executable, __file__))
67  if deps_file_path:
68    parsed_deps = parse_file_to_dict(deps_file_path)
69    if 'deps_os' in parsed_deps:
70      for deps_os in parsed_deps['deps_os']:
71        sys.stderr.write(' [%s]' % deps_os)
72  sys.stderr.write('\n\n')
73  sys.stderr.write(__doc__)
74
75
76def git_repository_sync_is_disabled(git, directory):
77  try:
78    disable = subprocess.check_output(
79      [git, 'config', 'sync-deps.disable'], cwd=directory)
80    return disable.lower().strip() in ['true', '1', 'yes', 'on']
81  except subprocess.CalledProcessError:
82    return False
83
84
85def is_git_toplevel(git, directory):
86  """Return true iff the directory is the top level of a Git repository.
87
88  Args:
89    git (string) the git executable
90
91    directory (string) the path into which the repository
92              is expected to be checked out.
93  """
94  try:
95    toplevel = subprocess.check_output(
96      [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
97    return os.path.realpath(directory) == os.path.realpath(toplevel.decode())
98  except subprocess.CalledProcessError:
99    return False
100
101
102def status(directory, commithash, change):
103  def truncate(s, length):
104    return s if len(s) <= length else s[:(length - 3)] + '...'
105  dlen = 36
106  directory = truncate(directory, dlen)
107  commithash = truncate(commithash, 40)
108  symbol = '>' if change else '@'
109  sys.stdout.write('%-*s %s %s\n' % (dlen, directory, symbol, commithash))
110
111
112def git_checkout_to_directory(git, repo, commithash, directory, verbose):
113  """Checkout (and clone if needed) a Git repository.
114
115  Args:
116    git (string) the git executable
117
118    repo (string) the location of the repository, suitable
119         for passing to `git clone`.
120
121    commithash (string) a commit, suitable for passing to `git checkout`
122
123    directory (string) the path into which the repository
124              should be checked out.
125
126    verbose (boolean)
127
128  Raises an exception if any calls to git fail.
129  """
130  if not os.path.isdir(directory):
131    subprocess.check_call(
132      [git, 'clone', '--quiet', '--no-checkout', repo, directory])
133    subprocess.check_call([git, 'checkout', '--quiet', commithash],
134                          cwd=directory)
135    if verbose:
136      status(directory, commithash, True)
137    return
138
139  if not is_git_toplevel(git, directory):
140    # if the directory exists, but isn't a git repo, you will modify
141    # the parent repostory, which isn't what you want.
142    sys.stdout.write('%s\n  IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
143    return
144
145  # Check to see if this repo is disabled.  Quick return.
146  if git_repository_sync_is_disabled(git, directory):
147    sys.stdout.write('%s\n  SYNC IS DISABLED.\n' % directory)
148    return
149
150  with open(os.devnull, 'w') as devnull:
151    # If this fails, we will fetch before trying again.  Don't spam user
152    # with error infomation.
153    if 0 == subprocess.call([git, 'checkout', '--quiet', commithash],
154                            cwd=directory, stderr=devnull):
155      # if this succeeds, skip slow `git fetch`.
156      if verbose:
157        status(directory, commithash, False)  # Success.
158      return
159
160  # If the repo has changed, always force use of the correct repo.
161  # If origin already points to repo, this is a quick no-op.
162  subprocess.check_call(
163      [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
164
165  subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
166
167  subprocess.check_call([git, 'checkout', '--quiet', commithash], cwd=directory)
168
169  if verbose:
170    status(directory, commithash, True)  # Success.
171
172
173def parse_file_to_dict(path):
174  dictionary = {}
175  with open(path) as f:
176    exec('def Var(x): return vars[x]\n' + f.read(), dictionary)
177  return dictionary
178
179
180def is_sha1_sum(s):
181  """SHA1 sums are 160 bits, encoded as lowercase hexadecimal."""
182  return len(s) == 40 and all(c in '0123456789abcdef' for c in s)
183
184
185def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
186  """Grab dependencies, with optional platform support.
187
188  Args:
189    deps_file_path (string) Path to the DEPS file.
190
191    command_line_os_requests (list of strings) Can be empty list.
192        List of strings that should each be a key in the deps_os
193        dictionary in the DEPS file.
194
195  Raises git Exceptions.
196  """
197  git = git_executable()
198  assert git
199
200  deps_file_directory = os.path.dirname(deps_file_path)
201  deps_file = parse_file_to_dict(deps_file_path)
202  dependencies = deps_file['deps'].copy()
203  os_specific_dependencies = deps_file.get('deps_os', dict())
204  if 'all' in command_line_os_requests:
205    for value in os_specific_dependencies.itervalues():
206      dependencies.update(value)
207  else:
208    for os_name in command_line_os_requests:
209      # Add OS-specific dependencies
210      if os_name in os_specific_dependencies:
211        dependencies.update(os_specific_dependencies[os_name])
212  for directory in dependencies:
213    for other_dir in dependencies:
214      if directory.startswith(other_dir + '/'):
215        raise Exception('%r is parent of %r' % (other_dir, directory))
216  list_of_arg_lists = []
217  for directory in sorted(dependencies):
218    if not isinstance(dependencies[directory], str):
219      if verbose:
220        sys.stdout.write( 'Skipping "%s".\n' % directory)
221      continue
222    if '@' in dependencies[directory]:
223      repo, commithash = dependencies[directory].split('@', 1)
224    else:
225      raise Exception("please specify commit")
226    if not is_sha1_sum(commithash):
227      raise Exception("poorly formed commit hash: %r" % commithash)
228
229    relative_directory = os.path.join(deps_file_directory, directory)
230
231    list_of_arg_lists.append(
232      (git, repo, commithash, relative_directory, verbose))
233
234  multithread(git_checkout_to_directory, list_of_arg_lists)
235
236
237def multithread(function, list_of_arg_lists):
238  # for args in list_of_arg_lists:
239  #   function(*args)
240  # return
241  threads = []
242  for args in list_of_arg_lists:
243    thread = threading.Thread(None, function, None, args)
244    thread.start()
245    threads.append(thread)
246  for thread in threads:
247    thread.join()
248
249
250def main(argv):
251  deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
252  verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
253
254  if '--help' in argv or '-h' in argv:
255    usage(deps_file_path)
256    return 1
257
258  git_sync_deps(deps_file_path, argv, verbose)
259  subprocess.check_call(
260      [sys.executable,
261       os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
262  return 0
263
264
265if __name__ == '__main__':
266  exit(main(sys.argv[1:]))
267