1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import os
7import tempfile
8
9import common
10from autotest_lib.client.bin import utils as common_utils
11from autotest_lib.client.common_lib import error
12from autotest_lib.client.common_lib.cros import dev_server
13from autotest_lib.client.common_lib.cros import retry
14from autotest_lib.server import utils as server_utils
15from autotest_lib.site_utils.lxc import constants
16
17try:
18    from chromite.lib import metrics
19except ImportError:
20    metrics = common_utils.metrics_mock
21
22
23def get_container_info(container_path, **filters):
24    """Get a collection of container information in the given container path.
25
26    This method parse the output of lxc-ls to get a list of container
27    information. The lxc-ls command output looks like:
28    NAME      STATE    IPV4       IPV6  AUTOSTART  PID   MEMORY  RAM     SWAP
29    --------------------------------------------------------------------------
30    base      STOPPED  -          -     NO         -     -       -       -
31    test_123  RUNNING  10.0.3.27  -     NO         8359  6.28MB  6.28MB  0.0MB
32
33    @param container_path: Path to look for containers.
34    @param filters: Key value to filter the containers, e.g., name='base'
35
36    @return: A list of dictionaries that each dictionary has the information of
37             a container. The keys are defined in ATTRIBUTES.
38    """
39    cmd = 'sudo lxc-ls -P %s -f -F %s' % (os.path.realpath(container_path),
40                                          ','.join(constants.ATTRIBUTES))
41    output = common_utils.run(cmd).stdout
42    info_collection = []
43
44    logging.info('cmd [%s] output:\n%s', cmd, output)
45
46    for line in output.splitlines()[1:]:
47        # Only LXC 1.x has the second line of '-' as a separator.
48        if line.startswith('------'):
49            continue
50        info_collection.append(dict(zip(constants.ATTRIBUTES, line.split())))
51    if filters:
52        filtered_collection = []
53        for key, value in filters.iteritems():
54            for info in info_collection:
55                if key in info and info[key] == value:
56                    filtered_collection.append(info)
57        info_collection = filtered_collection
58    return info_collection
59
60
61def download_extract(url, target, extract_dir):
62    """Download the file from given url and save it to the target, then extract.
63
64    @param url: Url to download the file.
65    @param target: Path of the file to save to.
66    @param extract_dir: Directory to extract the content of the file to.
67    """
68    remote_url = dev_server.DevServer.get_server_url(url)
69    # This can be run in multiple threads, pick a unique tmp_file.name.
70    with tempfile.NamedTemporaryFile(prefix=os.path.basename(target) + '_',
71                                     delete=False) as tmp_file:
72        if remote_url in dev_server.ImageServerBase.servers():
73            _download_via_devserver(url, tmp_file.name)
74        else:
75            _download_via_curl(url, tmp_file.name)
76        common_utils.run('sudo mv %s %s' % (tmp_file.name, target))
77    common_utils.run('sudo tar -xvf %s -C %s' % (target, extract_dir))
78
79
80# Make sure retries only happen in the non-timeout case.
81@retry.retry((error.CmdError),
82             raiselist=[error.CmdTimeoutError],
83             timeout_min=3*2,
84             delay_sec=10)
85def _download_via_curl(url, target_file_path):
86    # We do not want to retry on CmdTimeoutError but still retry on
87    # CmdError. Hence we can't use curl --timeout=...
88    common_utils.run('sudo curl -s %s -o %s' % (url, target_file_path),
89                     stderr_tee=common_utils.TEE_TO_LOGS, timeout=3*60)
90
91
92# Make sure retries only happen in the non-timeout case.
93@retry.retry((error.CmdError),
94             raiselist=[error.CmdTimeoutError],
95             timeout_min=(constants.DEVSERVER_CALL_TIMEOUT *
96                          constants.DEVSERVER_CALL_RETRY / 60),
97             delay_sec=constants.DEVSERVER_CALL_DELAY)
98def _download_via_devserver(url, target_file_path):
99    dev_server.ImageServerBase.download_file(
100        url, target_file_path, timeout=constants.DEVSERVER_CALL_TIMEOUT)
101
102
103def _install_package_precheck(packages):
104    """If SSP is not enabled or the test is running in chroot (using test_that),
105    packages installation should be skipped.
106
107    The check does not raise exception so tests started by test_that or running
108    in an Autotest setup with SSP disabled can continue. That assume the running
109    environment, chroot or a machine, has the desired packages installed
110    already.
111
112    @param packages: A list of names of the packages to install.
113
114    @return: True if package installation can continue. False if it should be
115             skipped.
116
117    """
118    if server_utils.is_inside_chroot():
119        logging.info('Test is running inside chroot. Install package %s is '
120                     'skipped.', packages)
121        return False
122
123    if not common_utils.is_in_container():
124        raise error.ContainerError('Package installation is only supported '
125                                   'when test is running inside container.')
126
127    return True
128
129
130def _remove_banned_packages(packages, banned_packages):
131    """Filter out packages.
132
133    @param packages: A set of packages names that have been requested.
134    @param items: A list of package names that are not to be installed.
135
136    @return: A sanatized set of packages names to install.
137    """
138    return {package for package in packages if package not in banned_packages}
139
140
141def _ensure_pip(target_setting):
142    """ Ensure pip is installed, if not install it.
143
144    @param target_setting: target command param specifying the path to where
145                           python packages should be installed.
146    """
147    try:
148        import pip
149    except ImportError:
150        common_utils.run(
151            'wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py')
152        common_utils.run('python /tmp/get-pip.py %s' % target_setting)
153
154
155@metrics.SecondsTimerDecorator(
156    '%s/install_packages_duration' % constants.STATS_KEY)
157@retry.retry(error.CmdError, timeout_min=30)
158def install_packages(packages=[], python_packages=[], force_latest=False):
159    """Install the given package inside container.
160
161    !!! WARNING !!!
162    This call may introduce several minutes of delay in test run. The best way
163    to avoid such delay is to update the base container used for the test run.
164    File a bug for infra deputy to update the base container with the new
165    package a test requires.
166
167    @param packages: A list of names of the packages to install.
168    @param python_packages: A list of names of the python packages to install
169                            using pip.
170    @param force_latest: True to force to install the latest version of the
171                         package. Default to False, which means skip installing
172                         the package if it's installed already, even with an old
173                         version.
174
175    @raise error.ContainerError: If package is attempted to be installed outside
176                                 a container.
177    @raise error.CmdError: If the package doesn't exist or failed to install.
178
179    """
180    if not _install_package_precheck(packages or python_packages):
181        return
182
183    # If force_latest is False, only install packages that are not already
184    # installed.
185    if not force_latest:
186        packages = [p for p in packages
187                    if not common_utils.is_package_installed(p)]
188        python_packages = [p for p in python_packages
189                           if not common_utils.is_python_package_installed(p)]
190        if not packages and not python_packages:
191            logging.debug(
192                'All packages are installed already, skip reinstall.')
193            return
194
195    # Always run apt-get update before installing any container. The base
196    # container may have outdated cache.
197    common_utils.run('sudo apt-get update')
198
199    # Make sure the lists are not None for iteration.
200    packages = [] if not packages else packages
201    # Remove duplicates.
202    packages = set(packages)
203
204    # Ubuntu distribution of pip is very old, do not use it as it causes
205    # segmentation faults.  Some tests request these packages, ensure they
206    # are not installed.
207    packages = _remove_banned_packages(packages, ['python-pip', 'python-dev'])
208
209    if packages:
210        common_utils.run(
211            'sudo DEBIAN_FRONTEND=noninteractive apt-get install %s -y '
212            '--force-yes' % ' '.join(packages))
213        logging.debug('Packages are installed: %s.', packages)
214
215    target_setting = ''
216    # For containers running in Moblab, /usr/local/lib/python2.7/dist-packages/
217    # is a readonly mount from the host. Therefore, new python modules have to
218    # be installed in /usr/lib/python2.7/dist-packages/
219    # Containers created in Moblab does not have autotest/site-packages folder.
220    if not os.path.exists('/usr/local/autotest/site-packages'):
221        target_setting = '--target="/usr/lib/python2.7/dist-packages/"'
222    # Pip should be installed in the base container, if not install it.
223    if python_packages:
224        _ensure_pip(target_setting)
225        common_utils.run('python -m pip install pip --upgrade')
226        common_utils.run('python -m pip install %s %s' % (target_setting,
227                                                          ' '.join(python_packages)))
228        logging.debug('Python packages are installed: %s.', python_packages)
229