1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""This module provides some tools to interact with LXC containers, for example: 6 1. Download base container from given GS location, setup the base container. 7 2. Create a snapshot as test container from base container. 8 3. Mount a directory in drone to the test container. 9 4. Run a command in the container and return the output. 10 5. Cleanup, e.g., destroy the container. 11 12This tool can also be used to set up a base container for test. For example, 13 python lxc.py -s -p /tmp/container 14This command will download and setup base container in directory /tmp/container. 15After that command finishes, you can run lxc command to work with the base 16container, e.g., 17 lxc-start -P /tmp/container -n base -d 18 lxc-attach -P /tmp/container -n base 19""" 20 21 22import argparse 23import logging 24import os 25import re 26import socket 27import sys 28import time 29 30import common 31from autotest_lib.client.bin import utils 32from autotest_lib.client.common_lib import error 33from autotest_lib.client.common_lib import global_config 34from autotest_lib.client.common_lib.cros import retry 35from autotest_lib.client.common_lib.cros.graphite import autotest_es 36from autotest_lib.client.common_lib.cros.graphite import autotest_stats 37from autotest_lib.server import utils as server_utils 38from autotest_lib.site_utils import lxc_config 39from autotest_lib.site_utils import lxc_utils 40 41 42config = global_config.global_config 43 44# Name of the base container. 45BASE = 'base' 46# Naming convention of test container, e.g., test_300_1422862512_2424, where: 47# 300: The test job ID. 48# 1422862512: The tick when container is created. 49# 2424: The PID of autoserv that starts the container. 50TEST_CONTAINER_NAME_FMT = 'test_%s_%d_%d' 51# Naming convention of the result directory in test container. 52RESULT_DIR_FMT = os.path.join(lxc_config.CONTAINER_AUTOTEST_DIR, 'results', 53 '%s') 54# Attributes to retrieve about containers. 55ATTRIBUTES = ['name', 'state'] 56 57# Format for mount entry to share a directory in host with container. 58# source is the directory in host, destination is the directory in container. 59# readonly is a binding flag for readonly mount, its value should be `,ro`. 60MOUNT_FMT = ('lxc.mount.entry = %(source)s %(destination)s none ' 61 'bind%(readonly)s 0 0') 62SSP_ENABLED = config.get_config_value('AUTOSERV', 'enable_ssp_container', 63 type=bool, default=True) 64# url to the base container. 65CONTAINER_BASE_URL = config.get_config_value('AUTOSERV', 'container_base') 66# Default directory used to store LXC containers. 67DEFAULT_CONTAINER_PATH = config.get_config_value('AUTOSERV', 'container_path') 68 69# Path to drone_temp folder in the container, which stores the control file for 70# test job to run. 71CONTROL_TEMP_PATH = os.path.join(lxc_config.CONTAINER_AUTOTEST_DIR, 'drone_tmp') 72 73# Bash command to return the file count in a directory. Test the existence first 74# so the command can return an error code if the directory doesn't exist. 75COUNT_FILE_CMD = '[ -d %(dir)s ] && ls %(dir)s | wc -l' 76 77# Command line to append content to a file 78APPEND_CMD_FMT = ('echo \'%(content)s\' | sudo tee --append %(file)s' 79 '> /dev/null') 80 81# Path to site-packates in Moblab 82MOBLAB_SITE_PACKAGES = '/usr/lib64/python2.7/site-packages' 83MOBLAB_SITE_PACKAGES_CONTAINER = '/usr/local/lib/python2.7/dist-packages/' 84 85# Flag to indicate it's running in a Moblab. Due to crbug.com/457496, lxc-ls has 86# different behavior in Moblab. 87IS_MOBLAB = utils.is_moblab() 88 89# TODO(dshi): If we are adding more logic in how lxc should interact with 90# different systems, we should consider code refactoring to use a setting-style 91# object to store following flags mapping to different systems. 92# TODO(crbug.com/464834): Snapshot clone is disabled until Moblab can 93# support overlayfs or aufs, which requires a newer kernel. 94SUPPORT_SNAPSHOT_CLONE = not IS_MOBLAB 95 96# Number of seconds to wait for network to be up in a container. 97NETWORK_INIT_TIMEOUT = 300 98# Network bring up is slower in Moblab. 99NETWORK_INIT_CHECK_INTERVAL = 2 if IS_MOBLAB else 0.1 100 101# Type string for container related metadata. 102CONTAINER_CREATE_METADB_TYPE = 'container_create' 103CONTAINER_CREATE_RETRY_METADB_TYPE = 'container_create_retry' 104CONTAINER_RUN_TEST_METADB_TYPE = 'container_run_test' 105 106STATS_KEY = 'lxc.%s' % socket.gethostname().replace('.', '_') 107timer = autotest_stats.Timer(STATS_KEY) 108# Timer used inside container should not include the hostname, as that will 109# create individual timer for each container. 110container_timer = autotest_stats.Timer('lxc') 111 112 113def _get_container_info_moblab(container_path, **filters): 114 """Get a collection of container information in the given container path 115 in a Moblab. 116 117 TODO(crbug.com/457496): remove this method once python 3 can be installed 118 in Moblab and lxc-ls command can use python 3 code. 119 120 When running in Moblab, lxc-ls behaves differently from a server with python 121 3 installed: 122 1. lxc-ls returns a list of containers installed under /etc/lxc, the default 123 lxc container directory. 124 2. lxc-ls --active lists all active containers, regardless where the 125 container is located. 126 For such differences, we have to special case Moblab to make the behavior 127 close to a server with python 3 installed. That is, 128 1. List only containers in a given folder. 129 2. Assume all active containers have state of RUNNING. 130 131 @param container_path: Path to look for containers. 132 @param filters: Key value to filter the containers, e.g., name='base' 133 134 @return: A list of dictionaries that each dictionary has the information of 135 a container. The keys are defined in ATTRIBUTES. 136 """ 137 info_collection = [] 138 active_containers = utils.run('sudo lxc-ls --active').stdout.split() 139 name_filter = filters.get('name', None) 140 state_filter = filters.get('state', None) 141 if filters and set(filters.keys()) - set(['name', 'state']): 142 raise error.ContainerError('When running in Moblab, container list ' 143 'filter only supports name and state.') 144 145 for name in os.listdir(container_path): 146 # Skip all files and folders without rootfs subfolder. 147 if (os.path.isfile(os.path.join(container_path, name)) or 148 not lxc_utils.path_exists(os.path.join(container_path, name, 149 'rootfs'))): 150 continue 151 info = {'name': name, 152 'state': 'RUNNING' if name in active_containers else 'STOPPED' 153 } 154 if ((name_filter and name_filter != info['name']) or 155 (state_filter and state_filter != info['state'])): 156 continue 157 158 info_collection.append(info) 159 return info_collection 160 161 162def get_container_info(container_path, **filters): 163 """Get a collection of container information in the given container path. 164 165 This method parse the output of lxc-ls to get a list of container 166 information. The lxc-ls command output looks like: 167 NAME STATE IPV4 IPV6 AUTOSTART PID MEMORY RAM SWAP 168 -------------------------------------------------------------------------- 169 base STOPPED - - NO - - - - 170 test_123 RUNNING 10.0.3.27 - NO 8359 6.28MB 6.28MB 0.0MB 171 172 @param container_path: Path to look for containers. 173 @param filters: Key value to filter the containers, e.g., name='base' 174 175 @return: A list of dictionaries that each dictionary has the information of 176 a container. The keys are defined in ATTRIBUTES. 177 """ 178 if IS_MOBLAB: 179 return _get_container_info_moblab(container_path, **filters) 180 181 cmd = 'sudo lxc-ls -P %s -f -F %s' % (os.path.realpath(container_path), 182 ','.join(ATTRIBUTES)) 183 output = utils.run(cmd).stdout 184 info_collection = [] 185 186 for line in output.splitlines()[2:]: 187 info_collection.append(dict(zip(ATTRIBUTES, line.split()))) 188 if filters: 189 filtered_collection = [] 190 for key, value in filters.iteritems(): 191 for info in info_collection: 192 if key in info and info[key] == value: 193 filtered_collection.append(info) 194 info_collection = filtered_collection 195 return info_collection 196 197 198def cleanup_if_fail(): 199 """Decorator to do cleanup if container fails to be set up. 200 """ 201 def deco_cleanup_if_fail(func): 202 """Wrapper for the decorator. 203 204 @param func: Function to be called. 205 """ 206 def func_cleanup_if_fail(*args, **kwargs): 207 """Decorator to do cleanup if container fails to be set up. 208 209 The first argument must be a ContainerBucket object, which can be 210 used to retrieve the container object by name. 211 212 @param func: function to be called. 213 @param args: arguments for function to be called. 214 @param kwargs: keyword arguments for function to be called. 215 """ 216 bucket = args[0] 217 name = utils.get_function_arg_value(func, 'name', args, kwargs) 218 try: 219 skip_cleanup = utils.get_function_arg_value( 220 func, 'skip_cleanup', args, kwargs) 221 except (KeyError, ValueError): 222 skip_cleanup = False 223 try: 224 return func(*args, **kwargs) 225 except: 226 exc_info = sys.exc_info() 227 try: 228 container = bucket.get(name) 229 if container and not skip_cleanup: 230 container.destroy() 231 except error.CmdError as e: 232 logging.error(e) 233 234 try: 235 job_id = utils.get_function_arg_value( 236 func, 'job_id', args, kwargs) 237 except (KeyError, ValueError): 238 job_id = '' 239 metadata={'drone': socket.gethostname(), 240 'job_id': job_id, 241 'success': False} 242 # Record all args if job_id is not available. 243 if not job_id: 244 metadata['args'] = str(args) 245 if kwargs: 246 metadata.update(kwargs) 247 autotest_es.post(use_http=True, 248 type_str=CONTAINER_CREATE_METADB_TYPE, 249 metadata=metadata) 250 251 # Raise the cached exception with original backtrace. 252 raise exc_info[0], exc_info[1], exc_info[2] 253 return func_cleanup_if_fail 254 return deco_cleanup_if_fail 255 256 257@retry.retry(error.CmdError, timeout_min=5) 258def download_extract(url, target, extract_dir): 259 """Download the file from given url and save it to the target, then extract. 260 261 @param url: Url to download the file. 262 @param target: Path of the file to save to. 263 @param extract_dir: Directory to extract the content of the file to. 264 """ 265 utils.run('sudo wget --timeout=300 -nv %s -O %s' % (url, target)) 266 utils.run('sudo tar -xvf %s -C %s' % (target, extract_dir)) 267 268 269def install_package_precheck(packages): 270 """If SSP is not enabled or the test is running in chroot (using test_that), 271 packages installation should be skipped. 272 273 The check does not raise exception so tests started by test_that or running 274 in an Autotest setup with SSP disabled can continue. That assume the running 275 environment, chroot or a machine, has the desired packages installed 276 already. 277 278 @param packages: A list of names of the packages to install. 279 280 @return: True if package installation can continue. False if it should be 281 skipped. 282 283 """ 284 if not SSP_ENABLED and not utils.is_in_container(): 285 logging.info('Server-side packaging is not enabled. Install package %s ' 286 'is skipped.', packages) 287 return False 288 289 if server_utils.is_inside_chroot(): 290 logging.info('Test is running inside chroot. Install package %s is ' 291 'skipped.', packages) 292 return False 293 294 return True 295 296 297@container_timer.decorate 298@retry.retry(error.CmdError, timeout_min=30) 299def install_packages(packages=[], python_packages=[]): 300 """Install the given package inside container. 301 302 @param packages: A list of names of the packages to install. 303 @param python_packages: A list of names of the python packages to install 304 using pip. 305 306 @raise error.ContainerError: If package is attempted to be installed outside 307 a container. 308 @raise error.CmdError: If the package doesn't exist or failed to install. 309 310 """ 311 if not install_package_precheck(packages or python_packages): 312 return 313 314 if not utils.is_in_container(): 315 raise error.ContainerError('Package installation is only supported ' 316 'when test is running inside container.') 317 # Always run apt-get update before installing any container. The base 318 # container may have outdated cache. 319 utils.run('sudo apt-get update') 320 # Make sure the lists are not None for iteration. 321 packages = [] if not packages else packages 322 if python_packages: 323 packages.extend(['python-pip', 'python-dev']) 324 if packages: 325 utils.run('sudo apt-get install %s -y --force-yes' % ' '.join(packages)) 326 logging.debug('Packages are installed: %s.', packages) 327 328 target_setting = '' 329 # For containers running in Moblab, /usr/local/lib/python2.7/dist-packages/ 330 # is a readonly mount from the host. Therefore, new python modules have to 331 # be installed in /usr/lib/python2.7/dist-packages/ 332 # Containers created in Moblab does not have autotest/site-packages folder. 333 if not os.path.exists('/usr/local/autotest/site-packages'): 334 target_setting = '--target="/usr/lib/python2.7/dist-packages/"' 335 if python_packages: 336 utils.run('sudo pip install %s %s' % (target_setting, 337 ' '.join(python_packages))) 338 logging.debug('Python packages are installed: %s.', python_packages) 339 340 341@container_timer.decorate 342@retry.retry(error.CmdError, timeout_min=20) 343def install_package(package): 344 """Install the given package inside container. 345 346 This function is kept for backwards compatibility reason. New code should 347 use function install_packages for better performance. 348 349 @param package: Name of the package to install. 350 351 @raise error.ContainerError: If package is attempted to be installed outside 352 a container. 353 @raise error.CmdError: If the package doesn't exist or failed to install. 354 355 """ 356 logging.warn('This function is obsoleted, please use install_packages ' 357 'instead.') 358 install_packages(packages=[package]) 359 360 361@container_timer.decorate 362@retry.retry(error.CmdError, timeout_min=20) 363def install_python_package(package): 364 """Install the given python package inside container using pip. 365 366 This function is kept for backwards compatibility reason. New code should 367 use function install_packages for better performance. 368 369 @param package: Name of the python package to install. 370 371 @raise error.CmdError: If the package doesn't exist or failed to install. 372 """ 373 logging.warn('This function is obsoleted, please use install_packages ' 374 'instead.') 375 install_packages(python_packages=[package]) 376 377 378class Container(object): 379 """A wrapper class of an LXC container. 380 381 The wrapper class provides methods to interact with a container, e.g., 382 start, stop, destroy, run a command. It also has attributes of the 383 container, including: 384 name: Name of the container. 385 state: State of the container, e.g., ABORTING, RUNNING, STARTING, STOPPED, 386 or STOPPING. 387 388 lxc-ls can also collect other attributes of a container including: 389 ipv4: IP address for IPv4. 390 ipv6: IP address for IPv6. 391 autostart: If the container will autostart at system boot. 392 pid: Process ID of the container. 393 memory: Memory used by the container, as a string, e.g., "6.2MB" 394 ram: Physical ram used by the container, as a string, e.g., "6.2MB" 395 swap: swap used by the container, as a string, e.g., "1.0MB" 396 397 For performance reason, such info is not collected for now. 398 399 The attributes available are defined in ATTRIBUTES constant. 400 """ 401 402 def __init__(self, container_path, attribute_values): 403 """Initialize an object of LXC container with given attribute values. 404 405 @param container_path: Directory that stores the container. 406 @param attribute_values: A dictionary of attribute values for the 407 container. 408 """ 409 self.container_path = os.path.realpath(container_path) 410 # Path to the rootfs of the container. This will be initialized when 411 # property rootfs is retrieved. 412 self._rootfs = None 413 for attribute, value in attribute_values.iteritems(): 414 setattr(self, attribute, value) 415 416 417 def refresh_status(self): 418 """Refresh the status information of the container. 419 """ 420 containers = get_container_info(self.container_path, name=self.name) 421 if not containers: 422 raise error.ContainerError( 423 'No container found in directory %s with name of %s.' % 424 self.container_path, self.name) 425 attribute_values = containers[0] 426 for attribute, value in attribute_values.iteritems(): 427 setattr(self, attribute, value) 428 429 430 @property 431 def rootfs(self): 432 """Path to the rootfs of the container. 433 434 This property returns the path to the rootfs of the container, that is, 435 the folder where the container stores its local files. It reads the 436 attribute lxc.rootfs from the config file of the container, e.g., 437 lxc.rootfs = /usr/local/autotest/containers/t4/rootfs 438 If the container is created with snapshot, the rootfs is a chain of 439 folders, separated by `:` and ordered by how the snapshot is created, 440 e.g., 441 lxc.rootfs = overlayfs:/usr/local/autotest/containers/base/rootfs: 442 /usr/local/autotest/containers/t4_s/delta0 443 This function returns the last folder in the chain, in above example, 444 that is `/usr/local/autotest/containers/t4_s/delta0` 445 446 Files in the rootfs will be accessible directly within container. For 447 example, a folder in host "[rootfs]/usr/local/file1", can be accessed 448 inside container by path "/usr/local/file1". Note that symlink in the 449 host can not across host/container boundary, instead, directory mount 450 should be used, refer to function mount_dir. 451 452 @return: Path to the rootfs of the container. 453 """ 454 if not self._rootfs: 455 cmd = ('sudo lxc-info -P %s -n %s -c lxc.rootfs' % 456 (self.container_path, self.name)) 457 lxc_rootfs_config = utils.run(cmd).stdout.strip() 458 match = re.match('lxc.rootfs = (.*)', lxc_rootfs_config) 459 if not match: 460 raise error.ContainerError( 461 'Failed to locate rootfs for container %s. lxc.rootfs ' 462 'in the container config file is %s' % 463 (self.name, lxc_rootfs_config)) 464 lxc_rootfs = match.group(1) 465 self.clone_from_snapshot = ':' in lxc_rootfs 466 if self.clone_from_snapshot: 467 self._rootfs = lxc_rootfs.split(':')[-1] 468 else: 469 self._rootfs = lxc_rootfs 470 return self._rootfs 471 472 473 def attach_run(self, command, bash=True): 474 """Attach to a given container and run the given command. 475 476 @param command: Command to run in the container. 477 @param bash: Run the command through bash -c "command". This allows 478 pipes to be used in command. Default is set to True. 479 480 @return: The output of the command. 481 482 @raise error.CmdError: If container does not exist, or not running. 483 """ 484 cmd = 'sudo lxc-attach -P %s -n %s' % (self.container_path, self.name) 485 if bash and not command.startswith('bash -c'): 486 command = 'bash -c "%s"' % utils.sh_escape(command) 487 cmd += ' -- %s' % command 488 # TODO(dshi): crbug.com/459344 Set sudo to default to False when test 489 # container can be unprivileged container. 490 return utils.run(cmd) 491 492 493 def is_network_up(self): 494 """Check if network is up in the container by curl base container url. 495 496 @return: True if the network is up, otherwise False. 497 """ 498 try: 499 self.attach_run('curl --head %s' % CONTAINER_BASE_URL) 500 return True 501 except error.CmdError as e: 502 logging.debug(e) 503 return False 504 505 506 @timer.decorate 507 def start(self, wait_for_network=True): 508 """Start the container. 509 510 @param wait_for_network: True to wait for network to be up. Default is 511 set to True. 512 513 @raise ContainerError: If container does not exist, or fails to start. 514 """ 515 cmd = 'sudo lxc-start -P %s -n %s -d' % (self.container_path, self.name) 516 output = utils.run(cmd).stdout 517 self.refresh_status() 518 if self.state != 'RUNNING': 519 raise error.ContainerError( 520 'Container %s failed to start. lxc command output:\n%s' % 521 (os.path.join(self.container_path, self.name), 522 output)) 523 524 if wait_for_network: 525 logging.debug('Wait for network to be up.') 526 start_time = time.time() 527 utils.poll_for_condition(condition=self.is_network_up, 528 timeout=NETWORK_INIT_TIMEOUT, 529 sleep_interval=NETWORK_INIT_CHECK_INTERVAL) 530 logging.debug('Network is up after %.2f seconds.', 531 time.time() - start_time) 532 533 534 @timer.decorate 535 def stop(self): 536 """Stop the container. 537 538 @raise ContainerError: If container does not exist, or fails to start. 539 """ 540 cmd = 'sudo lxc-stop -P %s -n %s' % (self.container_path, self.name) 541 output = utils.run(cmd).stdout 542 self.refresh_status() 543 if self.state != 'STOPPED': 544 raise error.ContainerError( 545 'Container %s failed to be stopped. lxc command output:\n' 546 '%s' % (os.path.join(self.container_path, self.name), 547 output)) 548 549 550 @timer.decorate 551 def destroy(self, force=True): 552 """Destroy the container. 553 554 @param force: Set to True to force to destroy the container even if it's 555 running. This is faster than stop a container first then 556 try to destroy it. Default is set to True. 557 558 @raise ContainerError: If container does not exist or failed to destroy 559 the container. 560 """ 561 cmd = 'sudo lxc-destroy -P %s -n %s' % (self.container_path, 562 self.name) 563 if force: 564 cmd += ' -f' 565 utils.run(cmd) 566 567 568 def mount_dir(self, source, destination, readonly=False): 569 """Mount a directory in host to a directory in the container. 570 571 @param source: Directory in host to be mounted. 572 @param destination: Directory in container to mount the source directory 573 @param readonly: Set to True to make a readonly mount, default is False. 574 """ 575 # Destination path in container must be relative. 576 destination = destination.lstrip('/') 577 # Create directory in container for mount. 578 utils.run('sudo mkdir -p %s' % os.path.join(self.rootfs, destination)) 579 config_file = os.path.join(self.container_path, self.name, 'config') 580 mount = MOUNT_FMT % {'source': source, 581 'destination': destination, 582 'readonly': ',ro' if readonly else ''} 583 utils.run(APPEND_CMD_FMT % {'content': mount, 'file': config_file}) 584 585 586 def verify_autotest_setup(self, job_id): 587 """Verify autotest code is set up properly in the container. 588 589 @param job_id: ID of the job, used to format job result folder. 590 591 @raise ContainerError: If autotest code is not set up properly. 592 """ 593 # Test autotest code is setup by verifying a list of 594 # (directory, minimum file count) 595 if IS_MOBLAB: 596 site_packages_path = MOBLAB_SITE_PACKAGES_CONTAINER 597 else: 598 site_packages_path = os.path.join(lxc_config.CONTAINER_AUTOTEST_DIR, 599 'site-packages') 600 directories_to_check = [ 601 (lxc_config.CONTAINER_AUTOTEST_DIR, 3), 602 (RESULT_DIR_FMT % job_id, 0), 603 (site_packages_path, 3)] 604 for directory, count in directories_to_check: 605 result = self.attach_run(command=(COUNT_FILE_CMD % 606 {'dir': directory})).stdout 607 logging.debug('%s entries in %s.', int(result), directory) 608 if int(result) < count: 609 raise error.ContainerError('%s is not properly set up.' % 610 directory) 611 612 613 def modify_import_order(self): 614 """Swap the python import order of lib and local/lib. 615 616 In Moblab, the host's python modules located in 617 /usr/lib64/python2.7/site-packages is mounted to following folder inside 618 container: /usr/local/lib/python2.7/dist-packages/. The modules include 619 an old version of requests module, which is used in autotest 620 site-packages. For test, the module is only used in 621 dev_server/symbolicate_dump for requests.call and requests.codes.OK. 622 When pip is installed inside the container, it installs requests module 623 with version of 2.2.1 in /usr/lib/python2.7/dist-packages/. The version 624 is newer than the one used in autotest site-packages, but not the latest 625 either. 626 According to /usr/lib/python2.7/site.py, modules in /usr/local/lib are 627 imported before the ones in /usr/lib. That leads to pip to use the older 628 version of requests (0.11.2), and it will fail. On the other hand, 629 requests module 2.2.1 can't be installed in CrOS (refer to CL:265759), 630 and higher version of requests module can't work with pip. 631 The only fix to resolve this is to switch the import order, so modules 632 in /usr/lib can be imported before /usr/local/lib. 633 """ 634 site_module = '/usr/lib/python2.7/site.py' 635 self.attach_run("sed -i ':a;N;$!ba;s/\"local\/lib\",\\n/" 636 "\"lib_placeholder\",\\n/g' %s" % site_module) 637 self.attach_run("sed -i ':a;N;$!ba;s/\"lib\",\\n/" 638 "\"local\/lib\",\\n/g' %s" % site_module) 639 self.attach_run('sed -i "s/lib_placeholder/lib/g" %s' % 640 site_module) 641 642 643 644class ContainerBucket(object): 645 """A wrapper class to interact with containers in a specific container path. 646 """ 647 648 def __init__(self, container_path=DEFAULT_CONTAINER_PATH): 649 """Initialize a ContainerBucket. 650 651 @param container_path: Path to the directory used to store containers. 652 Default is set to AUTOSERV/container_path in 653 global config. 654 """ 655 self.container_path = os.path.realpath(container_path) 656 657 658 def get_all(self): 659 """Get details of all containers. 660 661 @return: A dictionary of all containers with detailed attributes, 662 indexed by container name. 663 """ 664 info_collection = get_container_info(self.container_path) 665 containers = {} 666 for info in info_collection: 667 container = Container(self.container_path, info) 668 containers[container.name] = container 669 return containers 670 671 672 def get(self, name): 673 """Get a container with matching name. 674 675 @param name: Name of the container. 676 677 @return: A container object with matching name. Returns None if no 678 container matches the given name. 679 """ 680 return self.get_all().get(name, None) 681 682 683 def exist(self, name): 684 """Check if a container exists with the given name. 685 686 @param name: Name of the container. 687 688 @return: True if the container with the given name exists, otherwise 689 returns False. 690 """ 691 return self.get(name) != None 692 693 694 def destroy_all(self): 695 """Destroy all containers, base must be destroyed at the last. 696 """ 697 containers = self.get_all().values() 698 for container in sorted(containers, 699 key=lambda n: 1 if n.name == BASE else 0): 700 logging.info('Destroy container %s.', container.name) 701 container.destroy() 702 703 704 @timer.decorate 705 def create_from_base(self, name, disable_snapshot_clone=False, 706 force_cleanup=False): 707 """Create a container from the base container. 708 709 @param name: Name of the container. 710 @param disable_snapshot_clone: Set to True to force to clone without 711 using snapshot clone even if the host supports that. 712 @param force_cleanup: Force to cleanup existing container. 713 714 @return: A Container object for the created container. 715 716 @raise ContainerError: If the container already exist. 717 @raise error.CmdError: If lxc-clone call failed for any reason. 718 """ 719 if self.exist(name) and not force_cleanup: 720 raise error.ContainerError('Container %s already exists.' % name) 721 722 # Cleanup existing container with the given name. 723 container_folder = os.path.join(self.container_path, name) 724 if lxc_utils.path_exists(container_folder) and force_cleanup: 725 container = Container(self.container_path, {'name': name}) 726 try: 727 container.destroy() 728 except error.CmdError as e: 729 # The container could be created in a incompleted state. Delete 730 # the container folder instead. 731 logging.warn('Failed to destroy container %s, error: %s', 732 name, e) 733 utils.run('sudo rm -rf "%s"' % container_folder) 734 735 use_snapshot = SUPPORT_SNAPSHOT_CLONE and not disable_snapshot_clone 736 snapshot = '-s' if use_snapshot else '' 737 # overlayfs is the default clone backend storage. However it is not 738 # supported in Ganeti yet. Use aufs as the alternative. 739 aufs = '-B aufs' if utils.is_vm() and use_snapshot else '' 740 cmd = ('sudo lxc-clone -p %s -P %s %s' % 741 (self.container_path, self.container_path, 742 ' '.join([BASE, name, snapshot, aufs]))) 743 try: 744 utils.run(cmd) 745 return self.get(name) 746 except error.CmdError: 747 if not use_snapshot: 748 raise 749 else: 750 # Snapshot clone failed, retry clone without snapshot. The retry 751 # won't hit the code here and cause an infinite loop as 752 # disable_snapshot_clone is set to True. 753 container = self.create_from_base( 754 name, disable_snapshot_clone=True, force_cleanup=True) 755 # Report metadata about retry success. 756 autotest_es.post(use_http=True, 757 type_str=CONTAINER_CREATE_RETRY_METADB_TYPE, 758 metadata={'drone': socket.gethostname(), 759 'name': name, 760 'success': True}) 761 return container 762 763 764 @cleanup_if_fail() 765 def setup_base(self, name=BASE, force_delete=False): 766 """Setup base container. 767 768 @param name: Name of the base container, default to base. 769 @param force_delete: True to force to delete existing base container. 770 This action will destroy all running test 771 containers. Default is set to False. 772 """ 773 if not self.container_path: 774 raise error.ContainerError( 775 'You must set a valid directory to store containers in ' 776 'global config "AUTOSERV/ container_path".') 777 778 if not os.path.exists(self.container_path): 779 os.makedirs(self.container_path) 780 781 base_path = os.path.join(self.container_path, name) 782 if self.exist(name) and not force_delete: 783 logging.error( 784 'Base container already exists. Set force_delete to True ' 785 'to force to re-stage base container. Note that this ' 786 'action will destroy all running test containers') 787 # Set proper file permission. base container in moblab may have 788 # owner of not being root. Force to update the folder's owner. 789 # TODO(dshi): Change root to current user when test container can be 790 # unprivileged container. 791 utils.run('sudo chown -R root "%s"' % base_path) 792 utils.run('sudo chgrp -R root "%s"' % base_path) 793 return 794 795 # Destroy existing base container if exists. 796 if self.exist(name): 797 # TODO: We may need to destroy all snapshots created from this base 798 # container, not all container. 799 self.destroy_all() 800 801 # Download and untar the base container. 802 tar_path = os.path.join(self.container_path, '%s.tar.xz' % name) 803 path_to_cleanup = [tar_path, base_path] 804 for path in path_to_cleanup: 805 if os.path.exists(path): 806 utils.run('sudo rm -rf "%s"' % path) 807 download_extract(CONTAINER_BASE_URL, tar_path, self.container_path) 808 # Remove the downloaded container tar file. 809 utils.run('sudo rm "%s"' % tar_path) 810 # Set proper file permission. 811 # TODO(dshi): Change root to current user when test container can be 812 # unprivileged container. 813 utils.run('sudo chown -R root "%s"' % base_path) 814 utils.run('sudo chgrp -R root "%s"' % base_path) 815 816 # Update container config with container_path from global config. 817 config_path = os.path.join(base_path, 'config') 818 utils.run('sudo sed -i "s|container_dir|%s|g" "%s"' % 819 (self.container_path, config_path)) 820 821 822 @timer.decorate 823 @cleanup_if_fail() 824 def setup_test(self, name, job_id, server_package_url, result_path, 825 control=None, skip_cleanup=False): 826 """Setup test container for the test job to run. 827 828 The setup includes: 829 1. Install autotest_server package from given url. 830 2. Copy over local shadow_config.ini. 831 3. Mount local site-packages. 832 4. Mount test result directory. 833 834 TODO(dshi): Setup also needs to include test control file for autoserv 835 to run in container. 836 837 @param name: Name of the container. 838 @param job_id: Job id for the test job to run in the test container. 839 @param server_package_url: Url to download autotest_server package. 840 @param result_path: Directory to be mounted to container to store test 841 results. 842 @param control: Path to the control file to run the test job. Default is 843 set to None. 844 @param skip_cleanup: Set to True to skip cleanup, used to troubleshoot 845 container failures. 846 847 @return: A Container object for the test container. 848 849 @raise ContainerError: If container does not exist, or not running. 850 """ 851 start_time = time.time() 852 853 if not os.path.exists(result_path): 854 raise error.ContainerError('Result directory does not exist: %s', 855 result_path) 856 result_path = os.path.abspath(result_path) 857 858 # Create test container from the base container. 859 container = self.create_from_base(name) 860 861 # Deploy server side package 862 usr_local_path = os.path.join(container.rootfs, 'usr', 'local') 863 autotest_pkg_path = os.path.join(usr_local_path, 864 'autotest_server_package.tar.bz2') 865 autotest_path = os.path.join(usr_local_path, 'autotest') 866 # sudo is required so os.makedirs may not work. 867 utils.run('sudo mkdir -p %s'% usr_local_path) 868 869 download_extract(server_package_url, autotest_pkg_path, usr_local_path) 870 deploy_config_manager = lxc_config.DeployConfigManager(container) 871 deploy_config_manager.deploy_pre_start() 872 873 # Copy over control file to run the test job. 874 if control: 875 container_drone_temp = os.path.join(autotest_path, 'drone_tmp') 876 utils.run('sudo mkdir -p %s'% container_drone_temp) 877 container_control_file = os.path.join( 878 container_drone_temp, os.path.basename(control)) 879 utils.run('sudo cp %s %s' % (control, container_control_file)) 880 881 if IS_MOBLAB: 882 site_packages_path = MOBLAB_SITE_PACKAGES 883 site_packages_container_path = MOBLAB_SITE_PACKAGES_CONTAINER[1:] 884 else: 885 site_packages_path = os.path.join(common.autotest_dir, 886 'site-packages') 887 site_packages_container_path = os.path.join( 888 lxc_config.CONTAINER_AUTOTEST_DIR, 'site-packages') 889 mount_entries = [(site_packages_path, site_packages_container_path, 890 True), 891 (os.path.join(common.autotest_dir, 'puppylab'), 892 os.path.join(lxc_config.CONTAINER_AUTOTEST_DIR, 893 'puppylab'), 894 True), 895 (result_path, 896 os.path.join(RESULT_DIR_FMT % job_id), 897 False), 898 ] 899 # Update container config to mount directories. 900 for source, destination, readonly in mount_entries: 901 container.mount_dir(source, destination, readonly) 902 903 # Update file permissions. 904 # TODO(dshi): crbug.com/459344 Skip following action when test container 905 # can be unprivileged container. 906 utils.run('sudo chown -R root "%s"' % autotest_path) 907 utils.run('sudo chgrp -R root "%s"' % autotest_path) 908 909 container.start(name) 910 deploy_config_manager.deploy_post_start() 911 912 container.modify_import_order() 913 914 container.verify_autotest_setup(job_id) 915 916 autotest_es.post(use_http=True, 917 type_str=CONTAINER_CREATE_METADB_TYPE, 918 metadata={'drone': socket.gethostname(), 919 'job_id': job_id, 920 'time_used': time.time() - start_time, 921 'success': True}) 922 923 logging.debug('Test container %s is set up.', name) 924 return container 925 926 927def parse_options(): 928 """Parse command line inputs. 929 930 @raise argparse.ArgumentError: If command line arguments are invalid. 931 """ 932 parser = argparse.ArgumentParser() 933 parser.add_argument('-s', '--setup', action='store_true', 934 default=False, 935 help='Set up base container.') 936 parser.add_argument('-p', '--path', type=str, 937 help='Directory to store the container.', 938 default=DEFAULT_CONTAINER_PATH) 939 parser.add_argument('-f', '--force_delete', action='store_true', 940 default=False, 941 help=('Force to delete existing containers and rebuild ' 942 'base containers.')) 943 options = parser.parse_args() 944 if not options.setup and not options.force_delete: 945 raise argparse.ArgumentError( 946 'Use --setup to setup a base container, or --force_delete to ' 947 'delete all containers in given path.') 948 return options 949 950 951def main(): 952 """main script.""" 953 # Force to run the setup as superuser. 954 # TODO(dshi): crbug.com/459344 Set remove this enforcement when test 955 # container can be unprivileged container. 956 if utils.sudo_require_password(): 957 logging.warn('SSP requires root privilege to run commands, please ' 958 'grant root access to this process.') 959 utils.run('sudo true') 960 961 options = parse_options() 962 bucket = ContainerBucket(container_path=options.path) 963 if options.setup: 964 bucket.setup_base(force_delete=options.force_delete) 965 elif options.force_delete: 966 bucket.destroy_all() 967 968 969if __name__ == '__main__': 970 main() 971