1# Copyright 2020 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Mounts all the projects required by a selected Android target.
16
17For details on how filesystem overlays work see the filesystem overlays
18section of the README.md.
19"""
20
21from __future__ import absolute_import
22from __future__ import division
23from __future__ import print_function
24
25import collections
26import os
27import subprocess
28import tempfile
29import xml.etree.ElementTree as ET
30
31BindMount = collections.namedtuple('BindMount', ['source_dir', 'readonly'])
32
33
34class BindOverlay(object):
35  """Manages filesystem overlays of Android source tree using bind mounts.
36  """
37
38  MAX_BIND_MOUNTS = 10000
39
40  def _HideDir(self, target_dir):
41    """Temporarily replace the target directory for an empty directory.
42
43    Args:
44      target_dir: A string path to the target directory.
45
46    Returns:
47      A string path to the empty directory that replaced the target directory.
48    """
49    empty_dir = tempfile.mkdtemp(prefix='empty_dir_')
50    self._AddBindMount(empty_dir, target_dir)
51    return empty_dir
52
53  def _FindBindMountConflict(self, path):
54    """Finds any path in the bind mounts that conflicts with the provided path.
55
56    Args:
57      path: A string path to be checked.
58
59    Returns:
60      A string of the conflicting path in the bind mounts.
61      None if there was no conflict found.
62    """
63    conflict_path = None
64    for bind_destination, bind_mount in self._bind_mounts.items():
65      # Check if the path is a subdir or the bind destination
66      if path == bind_destination:
67        conflict_path = bind_mount.source_dir
68        break
69      elif path.startswith(bind_destination + os.sep):
70        relative_path = os.path.relpath(path, bind_destination)
71        path_in_source = os.path.join(bind_mount.source_dir, relative_path)
72        if os.path.exists(path_in_source) and os.listdir(path_in_source):
73          # A conflicting path exists within this bind mount
74          # and it's not empty
75          conflict_path = path_in_source
76          break
77
78    return conflict_path
79
80  def _AddOverlay(self, overlay_dir, intermediate_work_dir, skip_subdirs,
81                  destination_dir, rw_whitelist):
82    """Adds a single overlay directory.
83
84    Args:
85      overlay_dir: A string path to the overlay directory to apply.
86      intermediate_work_dir: A string path to the intermediate work directory used as the
87        base for constructing the overlay filesystem.
88      skip_subdirs: A set of string paths to skip from overlaying.
89      destination_dir: A string with the path to the source with the overlays
90        applied to it.
91      rw_whitelist: An optional set of source paths to bind mount with
92        read/write access.
93    """
94    # Traverse the overlay directory twice
95    # The first pass only process git projects
96    # The second time process all other files that are not in git projects
97
98    # We need to process all git projects first because
99    # the way we process a non-git directory will depend on if
100    # it contains a git project in a subdirectory or not.
101
102    dirs_with_git_projects = set('/')
103    for current_dir_origin, subdirs, files in os.walk(overlay_dir):
104
105      if current_dir_origin in skip_subdirs:
106        del subdirs[:]
107        continue
108
109      current_dir_relative = os.path.relpath(current_dir_origin, overlay_dir)
110      current_dir_destination = os.path.normpath(
111        os.path.join(destination_dir, current_dir_relative))
112
113      if '.git' in subdirs:
114        # The current dir is a git project
115        # so just bind mount it
116        del subdirs[:]
117
118        if rw_whitelist is None or current_dir_origin in rw_whitelist:
119          self._AddBindMount(current_dir_origin, current_dir_destination, False)
120        else:
121          self._AddBindMount(current_dir_origin, current_dir_destination, True)
122
123        current_dir_ancestor = current_dir_origin
124        while current_dir_ancestor and current_dir_ancestor not in dirs_with_git_projects:
125          dirs_with_git_projects.add(current_dir_ancestor)
126          current_dir_ancestor = os.path.dirname(current_dir_ancestor)
127
128    # Process all other files that are not in git projects
129    for current_dir_origin, subdirs, files in os.walk(overlay_dir):
130
131      if current_dir_origin in skip_subdirs:
132        del subdirs[:]
133        continue
134
135      if '.git' in subdirs:
136        del subdirs[:]
137        continue
138
139      current_dir_relative = os.path.relpath(current_dir_origin, overlay_dir)
140      current_dir_destination = os.path.normpath(
141        os.path.join(destination_dir, current_dir_relative))
142
143      if current_dir_origin in dirs_with_git_projects:
144        # Symbolic links to subdirectories
145        # have to be copied to the intermediate work directory.
146        # We can't bind mount them because bind mounts deference
147        # symbolic links, and the build system filters out any
148        # directory symbolic links.
149        for subdir in subdirs:
150          subdir_origin = os.path.join(current_dir_origin, subdir)
151          if os.path.islink(subdir_origin):
152            if subdir_origin not in skip_subdirs:
153              subdir_destination = os.path.join(intermediate_work_dir,
154                  current_dir_relative, subdir)
155              self._CopyFile(subdir_origin, subdir_destination)
156
157        # bind each file individually then keep travesting
158        for file in files:
159          file_origin = os.path.join(current_dir_origin, file)
160          file_destination = os.path.join(current_dir_destination, file)
161          if rw_whitelist is None or file_origin in rw_whitelist:
162            self._AddBindMount(file_origin, file_destination, False)
163          else:
164            self._AddBindMount(file_origin, file_destination, True)
165
166      else:
167        # The current dir does not have any git projects to it can be bind
168        # mounted wholesale
169        del subdirs[:]
170        if rw_whitelist is None or current_dir_origin in rw_whitelist:
171          self._AddBindMount(current_dir_origin, current_dir_destination, False)
172        else:
173          self._AddBindMount(current_dir_origin, current_dir_destination, True)
174
175  def _AddArtifactDirectories(self, source_dir, destination_dir, skip_subdirs):
176    """Add directories that were not synced as workspace source.
177
178    Args:
179      source_dir: A string with the path to the Android platform source.
180      destination_dir: A string with the path to the source where the overlays
181        will be applied.
182      skip_subdirs: A set of string paths to be skipped from overlays.
183
184    Returns:
185      A list of string paths to be skipped from overlaying.
186    """
187
188    # Ensure the main out directory exists
189    main_out_dir = os.path.join(source_dir, 'out')
190    if not os.path.exists(main_out_dir):
191      os.makedirs(main_out_dir)
192
193    for subdir in os.listdir(source_dir):
194      if subdir.startswith('out'):
195        out_origin = os.path.join(source_dir, subdir)
196        if out_origin in skip_subdirs:
197          continue
198        out_destination = os.path.join(destination_dir, subdir)
199        self._AddBindMount(out_origin, out_destination, False)
200        skip_subdirs.add(out_origin)
201
202    repo_origin = os.path.join(source_dir, '.repo')
203    if os.path.exists(repo_origin):
204      repo_destination = os.path.normpath(
205        os.path.join(destination_dir, '.repo'))
206      self._AddBindMount(repo_origin, repo_destination, False)
207      skip_subdirs.add(repo_origin)
208
209    return skip_subdirs
210
211  def _AddOverlays(self, source_dir, overlay_dirs, destination_dir,
212                   skip_subdirs, rw_whitelist):
213    """Add the selected overlay directories.
214
215    Args:
216      source_dir: A string with the path to the Android platform source.
217      overlay_dirs: A list of strings with the paths to the overlay
218        directory to apply.
219      destination_dir: A string with the path to the source where the overlays
220        will be applied.
221      skip_subdirs: A set of string paths to be skipped from overlays.
222      rw_whitelist: An optional set of source paths to bind mount with
223        read/write access.
224    """
225
226    # Create empty intermediate workdir
227    intermediate_work_dir = self._HideDir(destination_dir)
228    overlay_dirs.append(source_dir)
229
230    skip_subdirs = self._AddArtifactDirectories(source_dir, destination_dir,
231        skip_subdirs)
232
233
234    # Bind mount each overlay directory using a
235    # depth first traversal algorithm.
236    #
237    # The algorithm described works under the condition that the overlaid file
238    # systems do not have conflicting projects.
239    #
240    # The results of attempting to overlay two git projects on top
241    # of each other are unpredictable and may push the limits of bind mounts.
242
243    skip_subdirs.add(os.path.join(source_dir, 'overlays'))
244
245    for overlay_dir in overlay_dirs:
246      self._AddOverlay(overlay_dir, intermediate_work_dir,
247                       skip_subdirs, destination_dir, rw_whitelist)
248
249
250  def _AddBindMount(self, source_dir, destination_dir, readonly=False):
251    """Adds a bind mount for the specified directory.
252
253    Args:
254      source_dir: A string with the path of a source directory to bind.
255        It must already exist.
256      destination_dir: A string with the path ofa destination
257        directory to bind the source into. If it does not exist,
258        it will be created.
259      readonly: A flag to indicate whether this path should be bind mounted
260        with read-only access.
261    """
262    conflict_path = self._FindBindMountConflict(destination_dir)
263    if conflict_path:
264      raise ValueError("Project %s could not be overlaid at %s "
265        "because it conflicts with %s"
266        % (source_dir, destination_dir, conflict_path))
267
268    if len(self._bind_mounts) >= self.MAX_BIND_MOUNTS:
269      raise ValueError("Bind mount limit of %s reached" % self.MAX_BIND_MOUNTS)
270
271    self._bind_mounts[destination_dir] = BindMount(
272        source_dir=source_dir, readonly=readonly)
273
274  def _CopyFile(self, source_path, dest_path):
275    """Copies a file to the specified destination.
276
277    Args:
278      source_path: A string with the path of a source file to copy. It must
279        exist.
280      dest_path: A string with the path to copy the file to. It should not
281        exist.
282    """
283    dest_dir = os.path.dirname(dest_path)
284    if not os.path.exists(dest_dir):
285      os.makedirs(dest_dir)
286    subprocess.check_call(['cp', '--no-dereference', source_path, dest_path])
287
288  def GetBindMounts(self):
289    """Enumerates all bind mounts required by this Overlay.
290
291    Returns:
292      An ordered dict of BindMount objects keyed by destination path string.
293      The order of the bind mounts does matter, this is why it's an ordered
294      dict instead of a standard dict.
295    """
296    return self._bind_mounts
297
298  def __init__(self,
299               target,
300               source_dir,
301               config_file,
302               whiteout_list = [],
303               destination_dir=None,
304               rw_whitelist=None):
305    """Inits Overlay with the details of what is going to be overlaid.
306
307    Args:
308      target: A string with the name of the target to be prepared.
309      source_dir: A string with the path to the Android platform source.
310      config_file: A string path to the XML config file.
311      whiteout_list: A list of directories to hide from the build system.
312      destination_dir: A string with the path where the overlay filesystem
313        will be created. If none is provided, the overlay filesystem
314        will be applied directly on top of source_dir.
315      rw_whitelist: An optional set of source paths to bind mount with
316        read/write access. If none is provided, all paths will be mounted with
317        read/write access. If the set is empty, all paths will be mounted
318        read-only.
319    """
320
321    if not destination_dir:
322      destination_dir = source_dir
323
324    self._overlay_dirs = None
325    # The order of the bind mounts does matter, this is why it's an ordered
326    # dict instead of a standard dict.
327    self._bind_mounts = collections.OrderedDict()
328
329    # We will be repeateadly searching for items to skip so a set
330    # seems appropriate
331    skip_subdirs = set(whiteout_list)
332
333    # The read/write whitelist provids paths relative to the source dir. It
334    # needs to be updated with absolute paths to make lookup possible.
335    if rw_whitelist:
336      rw_whitelist = {os.path.join(source_dir, p) for p in rw_whitelist}
337
338    overlay_dirs = []
339    overlay_map = get_overlay_map(config_file)
340    for overlay_dir in overlay_map[target]:
341      overlay_dir = os.path.join(source_dir, 'overlays', overlay_dir)
342      overlay_dirs.append(overlay_dir)
343
344    self._AddOverlays(
345        source_dir, overlay_dirs, destination_dir, skip_subdirs, rw_whitelist)
346
347    # If specified for this target, create a custom filesystem view
348    fs_view_map = get_fs_view_map(config_file)
349    if target in fs_view_map:
350      for path_relative_from, path_relative_to in fs_view_map[target]:
351        path_from = os.path.join(source_dir, path_relative_from)
352        if os.path.isfile(path_from) or os.path.isdir(path_from):
353          path_to = os.path.join(destination_dir, path_relative_to)
354          if rw_whitelist is None or path_from in rw_whitelist:
355            self._AddBindMount(path_from, path_to, False)
356          else:
357            self._AddBindMount(path_from, path_to, True)
358        else:
359          raise ValueError("Path '%s' must be a file or directory" % path_from)
360
361    self._overlay_dirs = overlay_dirs
362    print('Applied overlays ' + ' '.join(self._overlay_dirs))
363
364  def __del__(self):
365    """Cleans up Overlay.
366    """
367    if self._overlay_dirs:
368      print('Stripped out overlay ' + ' '.join(self._overlay_dirs))
369
370def get_config(config_file):
371  """Parses the overlay configuration file.
372
373  Args:
374    config_file: A string path to the XML config file.
375
376  Returns:
377    A root config XML Element.
378    None if there is no config file.
379  """
380  config = None
381  if os.path.exists(config_file):
382    tree = ET.parse(config_file)
383    config = tree.getroot()
384  return config
385
386def get_overlay_map(config_file):
387  """Retrieves the map of overlays for each target.
388
389  Args:
390    config_file: A string path to the XML config file.
391
392  Returns:
393    A dict of keyed by target name. Each value in the
394    dict is a list of overlay names corresponding to
395    the target.
396  """
397  overlay_map = {}
398  config = get_config(config_file)
399  # The presence of the config file is optional
400  if config:
401    for target in config.findall('target'):
402      name = target.get('name')
403      overlay_list = [o.get('name') for o in target.findall('overlay')]
404      overlay_map[name] = overlay_list
405    # A valid configuration file is required
406    # to have at least one overlay target
407    if not overlay_map:
408      raise ValueError('Error: the overlay configuration file '
409          'is missing at least one overlay target')
410
411  return overlay_map
412
413def get_fs_view_map(config_file):
414  """Retrieves the map of filesystem views for each target.
415
416  Args:
417    config_file: A string path to the XML config file.
418
419  Returns:
420    A dict of filesystem views keyed by target name.
421    A filesystem view is a list of (source, destination)
422    string path tuples.
423  """
424  fs_view_map = {}
425  config = get_config(config_file)
426
427  # The presence of the config file is optional
428  if config:
429    # A valid config file is not required to
430    # include FS Views, only overlay targets
431    views = {}
432    for view in config.findall('view'):
433      name = view.get('name')
434      paths = []
435      for path in view.findall('path'):
436        paths.append((
437              path.get('source'),
438              path.get('destination')))
439      views[name] = paths
440
441    for target in config.findall('target'):
442      target_name = target.get('name')
443      view_paths = []
444      for view in target.findall('view'):
445        view_paths.extend(views[view.get('name')])
446
447      if view_paths:
448        fs_view_map[target_name] = view_paths
449
450  return fs_view_map
451