1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import collections 6import json 7import logging 8import os 9import re 10import tempfile 11import time 12 13import common 14from autotest_lib.client.bin import utils 15from autotest_lib.client.common_lib import error 16from autotest_lib.site_utils.lxc import constants 17from autotest_lib.site_utils.lxc import lxc 18from autotest_lib.site_utils.lxc import utils as lxc_utils 19 20try: 21 from chromite.lib import metrics 22except ImportError: 23 metrics = utils.metrics_mock 24 25ISOLATESERVER = 'https://isolateserver.appspot.com' 26 27# Naming convention of test container, e.g., test_300_1422862512_2424, where: 28# 300: The test job ID. 29# 1422862512: The tick when container is created. 30# 2424: The PID of autoserv that starts the container. 31_TEST_CONTAINER_NAME_FMT = 'test_%s_%d_%d' 32# Name of the container ID file. 33_CONTAINER_ID_FILENAME = 'container_id.json' 34 35 36class ContainerId(collections.namedtuple('ContainerId', 37 ['job_id', 'creation_time', 'pid'])): 38 """An identifier for containers.""" 39 40 # Optimization. Avoids __dict__ creation. Empty because this subclass has 41 # no instance vars of its own. 42 __slots__ = () 43 44 45 def __str__(self): 46 return _TEST_CONTAINER_NAME_FMT % self 47 48 49 def save(self, path): 50 """Saves the ID to the given path. 51 52 @param path: Path to a directory where the container ID will be 53 serialized. 54 """ 55 dst = os.path.join(path, _CONTAINER_ID_FILENAME) 56 with open(dst, 'w') as f: 57 json.dump(self, f) 58 59 @classmethod 60 def load(cls, path): 61 """Reads the ID from the given path. 62 63 @param path: Path to check for a serialized container ID. 64 65 @return: A container ID if one is found on the given path, or None 66 otherwise. 67 68 @raise ValueError: If a JSON load error occurred. 69 @raise TypeError: If the file was valid JSON but didn't contain a valid 70 ContainerId. 71 """ 72 src = os.path.join(path, _CONTAINER_ID_FILENAME) 73 74 try: 75 with open(src, 'r') as f: 76 job_id, ctime, pid = json.load(f) 77 except IOError: 78 # File not found, or couldn't be opened for some other reason. 79 # Treat all these cases as no ID. 80 return None 81 # TODO(pprabhu, crbug.com/842343) Remove this once all persistent 82 # container ids have migrated to str. 83 job_id = str(job_id) 84 return cls(job_id, ctime, pid) 85 86 87 @classmethod 88 def create(cls, job_id, ctime=None, pid=None): 89 """Creates a new container ID. 90 91 @param job_id: The first field in the ID. 92 @param ctime: The second field in the ID. Optional. If not provided, 93 the current epoch timestamp is used. 94 @param pid: The third field in the ID. Optional. If not provided, the 95 PID of the current process is used. 96 """ 97 if ctime is None: 98 ctime = int(time.time()) 99 if pid is None: 100 pid = os.getpid() 101 # TODO(pprabhu) Drop str() cast once 102 # job_directories.get_job_id_or_task_id() starts returning str directly. 103 return cls(str(job_id), ctime, pid) 104 105 106class Container(object): 107 """A wrapper class of an LXC container. 108 109 The wrapper class provides methods to interact with a container, e.g., 110 start, stop, destroy, run a command. It also has attributes of the 111 container, including: 112 name: Name of the container. 113 state: State of the container, e.g., ABORTING, RUNNING, STARTING, STOPPED, 114 or STOPPING. 115 116 lxc-ls can also collect other attributes of a container including: 117 ipv4: IP address for IPv4. 118 ipv6: IP address for IPv6. 119 autostart: If the container will autostart at system boot. 120 pid: Process ID of the container. 121 memory: Memory used by the container, as a string, e.g., "6.2MB" 122 ram: Physical ram used by the container, as a string, e.g., "6.2MB" 123 swap: swap used by the container, as a string, e.g., "1.0MB" 124 125 For performance reason, such info is not collected for now. 126 127 The attributes available are defined in ATTRIBUTES constant. 128 """ 129 130 _LXC_VERSION = None 131 132 def __init__(self, container_path, name, attribute_values, src=None, 133 snapshot=False): 134 """Initialize an object of LXC container with given attribute values. 135 136 @param container_path: Directory that stores the container. 137 @param name: Name of the container. 138 @param attribute_values: A dictionary of attribute values for the 139 container. 140 @param src: An optional source container. If provided, the source 141 continer is cloned, and the new container will point to the 142 clone. 143 @param snapshot: If a source container was specified, this argument 144 specifies whether or not to create a snapshot clone. 145 The default is to attempt to create a snapshot. 146 If a snapshot is requested and creating the snapshot 147 fails, a full clone will be attempted. 148 """ 149 self.container_path = os.path.realpath(container_path) 150 # Path to the rootfs of the container. This will be initialized when 151 # property rootfs is retrieved. 152 self._rootfs = None 153 self.name = name 154 for attribute, value in attribute_values.iteritems(): 155 setattr(self, attribute, value) 156 157 # Clone the container 158 if src is not None: 159 # Clone the source container to initialize this one. 160 lxc_utils.clone(src.container_path, src.name, self.container_path, 161 self.name, snapshot) 162 # Newly cloned containers have no ID. 163 self._id = None 164 else: 165 # This may be an existing container. Try to read the ID. 166 try: 167 self._id = ContainerId.load( 168 os.path.join(self.container_path, self.name)) 169 except (ValueError, TypeError): 170 # Ignore load errors. ContainerBucket currently queries every 171 # container quite frequently, and emitting exceptions here would 172 # cause any invalid containers on a server to block all 173 # ContainerBucket.get_all calls (see crbug/783865). 174 logging.warning('Unable to determine ID for container %s:', 175 self.name) 176 self._id = None 177 178 if not Container._LXC_VERSION: 179 Container._LXC_VERSION = lxc_utils.get_lxc_version() 180 181 182 @classmethod 183 def create_from_existing_dir(cls, lxc_path, name, **kwargs): 184 """Creates a new container instance for an lxc container that already 185 exists on disk. 186 187 @param lxc_path: The LXC path for the container. 188 @param name: The container name. 189 190 @raise error.ContainerError: If the container doesn't already exist. 191 192 @return: The new container. 193 """ 194 return cls(lxc_path, name, kwargs) 195 196 197 # Containers have a name and an ID. The name is simply the name of the LXC 198 # container. The ID is the actual key that is used to identify the 199 # container to the autoserv system. In the case of a JIT-created container, 200 # we have the ID at the container's creation time so we use that to name the 201 # container. This may not be the case for other types of containers. 202 @classmethod 203 def clone(cls, src, new_name=None, new_path=None, snapshot=False, 204 cleanup=False): 205 """Creates a clone of this container. 206 207 @param src: The original container. 208 @param new_name: Name for the cloned container. If this is not 209 provided, a random unique container name will be 210 generated. 211 @param new_path: LXC path for the cloned container (optional; if not 212 specified, the new container is created in the same 213 directory as the source container). 214 @param snapshot: Whether to snapshot, or create a full clone. Note that 215 snapshot cloning is not supported on all platforms. If 216 this code is running on a platform that does not 217 support snapshot clones, this flag is ignored. 218 @param cleanup: If a container with the given name and path already 219 exist, clean it up first. 220 """ 221 if new_path is None: 222 new_path = src.container_path 223 224 if new_name is None: 225 _, new_name = os.path.split( 226 tempfile.mkdtemp(dir=new_path, prefix='container.')) 227 logging.debug('Generating new name for container: %s', new_name) 228 else: 229 # If a container exists at this location, clean it up first 230 container_folder = os.path.join(new_path, new_name) 231 if lxc_utils.path_exists(container_folder): 232 if not cleanup: 233 raise error.ContainerError('Container %s already exists.' % 234 new_name) 235 container = Container.create_from_existing_dir(new_path, 236 new_name) 237 try: 238 container.destroy() 239 except error.CmdError as e: 240 # The container could be created in a incompleted 241 # state. Delete the container folder instead. 242 logging.warn('Failed to destroy container %s, error: %s', 243 new_name, e) 244 utils.run('sudo rm -rf "%s"' % container_folder) 245 # Create the directory prior to creating the new container. This 246 # puts the ownership of the container under the current process's 247 # user, rather than root. This is necessary to enable the 248 # ContainerId to serialize properly. 249 os.mkdir(container_folder) 250 251 # Create and return the new container. 252 new_container = cls(new_path, new_name, {}, src, snapshot) 253 254 return new_container 255 256 257 def refresh_status(self): 258 """Refresh the status information of the container. 259 """ 260 containers = lxc.get_container_info(self.container_path, name=self.name) 261 if not containers: 262 raise error.ContainerError( 263 'No container found in directory %s with name of %s.' % 264 (self.container_path, self.name)) 265 attribute_values = containers[0] 266 for attribute, value in attribute_values.iteritems(): 267 setattr(self, attribute, value) 268 269 270 @property 271 def rootfs(self): 272 """Path to the rootfs of the container. 273 274 This property returns the path to the rootfs of the container, that is, 275 the folder where the container stores its local files. It reads the 276 attribute lxc.rootfs from the config file of the container, e.g., 277 lxc.rootfs = /usr/local/autotest/containers/t4/rootfs 278 If the container is created with snapshot, the rootfs is a chain of 279 folders, separated by `:` and ordered by how the snapshot is created, 280 e.g., 281 lxc.rootfs = overlayfs:/usr/local/autotest/containers/base/rootfs: 282 /usr/local/autotest/containers/t4_s/delta0 283 This function returns the last folder in the chain, in above example, 284 that is `/usr/local/autotest/containers/t4_s/delta0` 285 286 Files in the rootfs will be accessible directly within container. For 287 example, a folder in host "[rootfs]/usr/local/file1", can be accessed 288 inside container by path "/usr/local/file1". Note that symlink in the 289 host can not across host/container boundary, instead, directory mount 290 should be used, refer to function mount_dir. 291 292 @return: Path to the rootfs of the container. 293 """ 294 lxc_rootfs_config_name = 'lxc.rootfs' 295 # Check to see if the major lxc version is 3 or greater 296 if Container._LXC_VERSION: 297 logging.info("Detected lxc version %s", Container._LXC_VERSION) 298 if Container._LXC_VERSION[0] >= 3: 299 lxc_rootfs_config_name = 'lxc.rootfs.path' 300 if not self._rootfs: 301 lxc_rootfs = self._get_lxc_config(lxc_rootfs_config_name)[0] 302 cloned_from_snapshot = ':' in lxc_rootfs 303 if cloned_from_snapshot: 304 self._rootfs = lxc_rootfs.split(':')[-1] 305 else: 306 self._rootfs = lxc_rootfs 307 return self._rootfs 308 309 310 def attach_run(self, command, bash=True): 311 """Attach to a given container and run the given command. 312 313 @param command: Command to run in the container. 314 @param bash: Run the command through bash -c "command". This allows 315 pipes to be used in command. Default is set to True. 316 317 @return: The output of the command. 318 319 @raise error.CmdError: If container does not exist, or not running. 320 """ 321 cmd = 'sudo lxc-attach -P %s -n %s' % (self.container_path, self.name) 322 if bash and not command.startswith('bash -c'): 323 command = 'bash -c "%s"' % utils.sh_escape(command) 324 cmd += ' -- %s' % command 325 # TODO(dshi): crbug.com/459344 Set sudo to default to False when test 326 # container can be unprivileged container. 327 return utils.run(cmd) 328 329 330 def is_network_up(self): 331 """Check if network is up in the container by curl base container url. 332 333 @return: True if the network is up, otherwise False. 334 """ 335 try: 336 self.attach_run('curl --head %s' % constants.CONTAINER_BASE_URL) 337 return True 338 except error.CmdError as e: 339 logging.debug(e) 340 return False 341 342 343 @metrics.SecondsTimerDecorator( 344 '%s/container_start_duration' % constants.STATS_KEY) 345 def start(self, wait_for_network=True): 346 """Start the container. 347 348 @param wait_for_network: True to wait for network to be up. Default is 349 set to True. 350 351 @raise ContainerError: If container does not exist, or fails to start. 352 """ 353 cmd = 'sudo lxc-start -P %s -n %s -d' % (self.container_path, self.name) 354 output = utils.run(cmd).stdout 355 if not self.is_running(): 356 raise error.ContainerError( 357 'Container %s failed to start. lxc command output:\n%s' % 358 (os.path.join(self.container_path, self.name), 359 output)) 360 361 if wait_for_network: 362 logging.debug('Wait for network to be up.') 363 start_time = time.time() 364 utils.poll_for_condition( 365 condition=self.is_network_up, 366 timeout=constants.NETWORK_INIT_TIMEOUT, 367 sleep_interval=constants.NETWORK_INIT_CHECK_INTERVAL, 368 desc='network is up') 369 logging.debug('Network is up after %.2f seconds.', 370 time.time() - start_time) 371 372 373 @metrics.SecondsTimerDecorator( 374 '%s/container_stop_duration' % constants.STATS_KEY) 375 def stop(self): 376 """Stop the container. 377 378 @raise ContainerError: If container does not exist, or fails to start. 379 """ 380 cmd = 'sudo lxc-stop -P %s -n %s' % (self.container_path, self.name) 381 output = utils.run(cmd).stdout 382 self.refresh_status() 383 if self.state != 'STOPPED': 384 raise error.ContainerError( 385 'Container %s failed to be stopped. lxc command output:\n' 386 '%s' % (os.path.join(self.container_path, self.name), 387 output)) 388 389 390 @metrics.SecondsTimerDecorator( 391 '%s/container_destroy_duration' % constants.STATS_KEY) 392 def destroy(self, force=True): 393 """Destroy the container. 394 395 @param force: Set to True to force to destroy the container even if it's 396 running. This is faster than stop a container first then 397 try to destroy it. Default is set to True. 398 399 @raise ContainerError: If container does not exist or failed to destroy 400 the container. 401 """ 402 logging.debug('Destroying container %s/%s', 403 self.container_path, 404 self.name) 405 lxc_utils.destroy(self.container_path, self.name, force=force) 406 407 408 def mount_dir(self, source, destination, readonly=False): 409 """Mount a directory in host to a directory in the container. 410 411 @param source: Directory in host to be mounted. 412 @param destination: Directory in container to mount the source directory 413 @param readonly: Set to True to make a readonly mount, default is False. 414 """ 415 # Destination path in container must be relative. 416 destination = destination.lstrip('/') 417 # Create directory in container for mount. Changes to container rootfs 418 # require sudo. 419 utils.run('sudo mkdir -p %s' % os.path.join(self.rootfs, destination)) 420 mount = ('%s %s none bind%s 0 0' % 421 (source, destination, ',ro' if readonly else '')) 422 self._set_lxc_config('lxc.mount.entry', mount) 423 424 def verify_autotest_setup(self, job_folder): 425 """Verify autotest code is set up properly in the container. 426 427 @param job_folder: Name of the job result folder. 428 429 @raise ContainerError: If autotest code is not set up properly. 430 """ 431 # Test autotest code is setup by verifying a list of 432 # (directory, minimum file count) 433 directories_to_check = [ 434 (constants.CONTAINER_AUTOTEST_DIR, 3), 435 (constants.RESULT_DIR_FMT % job_folder, 0), 436 (constants.CONTAINER_SITE_PACKAGES_PATH, 3)] 437 for directory, count in directories_to_check: 438 result = self.attach_run(command=(constants.COUNT_FILE_CMD % 439 {'dir': directory})).stdout 440 logging.debug('%s entries in %s.', int(result), directory) 441 if int(result) < count: 442 raise error.ContainerError('%s is not properly set up.' % 443 directory) 444 # lxc-attach and run command does not run in shell, thus .bashrc is not 445 # loaded. Following command creates a symlink in /usr/bin/ for gsutil 446 # if it's installed. 447 # TODO(dshi): Remove this code after lab container is updated with 448 # gsutil installed in /usr/bin/ 449 self.attach_run('test -f /root/gsutil/gsutil && ' 450 'ln -s /root/gsutil/gsutil /usr/bin/gsutil || true') 451 452 453 def modify_import_order(self): 454 """Swap the python import order of lib and local/lib. 455 456 In Moblab, the host's python modules located in 457 /usr/lib64/python2.7/site-packages is mounted to following folder inside 458 container: /usr/local/lib/python2.7/dist-packages/. The modules include 459 an old version of requests module, which is used in autotest 460 site-packages. For test, the module is only used in 461 dev_server/symbolicate_dump for requests.call and requests.codes.OK. 462 When pip is installed inside the container, it installs requests module 463 with version of 2.2.1 in /usr/lib/python2.7/dist-packages/. The version 464 is newer than the one used in autotest site-packages, but not the latest 465 either. 466 According to /usr/lib/python2.7/site.py, modules in /usr/local/lib are 467 imported before the ones in /usr/lib. That leads to pip to use the older 468 version of requests (0.11.2), and it will fail. On the other hand, 469 requests module 2.2.1 can't be installed in CrOS (refer to CL:265759), 470 and higher version of requests module can't work with pip. 471 The only fix to resolve this is to switch the import order, so modules 472 in /usr/lib can be imported before /usr/local/lib. 473 """ 474 site_module = '/usr/lib/python2.7/site.py' 475 self.attach_run("sed -i ':a;N;$!ba;s/\"local\/lib\",\\n/" 476 "\"lib_placeholder\",\\n/g' %s" % site_module) 477 self.attach_run("sed -i ':a;N;$!ba;s/\"lib\",\\n/" 478 "\"local\/lib\",\\n/g' %s" % site_module) 479 self.attach_run('sed -i "s/lib_placeholder/lib/g" %s' % 480 site_module) 481 482 483 def is_running(self): 484 """Returns whether or not this container is currently running.""" 485 self.refresh_status() 486 return self.state == 'RUNNING' 487 488 489 def set_hostname(self, hostname): 490 """Sets the hostname within the container. 491 492 This method can only be called on a running container. 493 494 @param hostname The new container hostname. 495 496 @raise ContainerError: If the container is not running. 497 """ 498 if not self.is_running(): 499 raise error.ContainerError( 500 'set_hostname can only be called on running containers.') 501 502 self.attach_run('hostname %s' % (hostname)) 503 self.attach_run(constants.APPEND_CMD_FMT % { 504 'content': '127.0.0.1 %s' % (hostname), 505 'file': '/etc/hosts'}) 506 507 508 def install_ssp(self, ssp_url): 509 """Downloads and installs the given server package. 510 511 @param ssp_url: The URL of the ssp to download and install. 512 """ 513 usr_local_path = os.path.join(self.rootfs, 'usr', 'local') 514 autotest_pkg_path = os.path.join(usr_local_path, 515 'autotest_server_package.tar.bz2') 516 # Changes within the container rootfs require sudo. 517 utils.run('sudo mkdir -p %s'% usr_local_path) 518 519 lxc.download_extract(ssp_url, autotest_pkg_path, usr_local_path) 520 521 def install_ssp_isolate(self, isolate_hash, dest_path=None): 522 """Downloads and install the contents of the given isolate. 523 This places the isolate contents under /usr/local or a provided path. 524 Most commonly this is a copy of a specific autotest version, in which 525 case: 526 /usr/local/autotest contains the autotest code 527 /usr/local/logs contains logs from the installation process. 528 529 @param isolate_hash: The hash string which serves as a key to retrieve 530 the desired isolate 531 @param dest_path: Path to the directory to place the isolate in. 532 Defaults to /usr/local/ 533 534 @return: Exit status of the installation command. 535 """ 536 dest_path = dest_path or os.path.join(self.rootfs, 'usr', 'local') 537 isolate_log_path = os.path.join( 538 self.rootfs, 'usr', 'local', 'logs', 'isolate') 539 log_file = os.path.join(isolate_log_path, 540 'contents.' + time.strftime('%Y-%m-%d-%H.%M.%S')) 541 542 utils.run('sudo mkdir -p %s' % isolate_log_path) 543 _command = ("sudo isolated download -isolated {sha} -I {server}" 544 " -output-dir {dest_dir} -output-files {log_file}") 545 546 return utils.run(_command.format( 547 sha=isolate_hash, dest_dir=dest_path, 548 log_file=log_file, server=ISOLATESERVER)) 549 550 551 def install_control_file(self, control_file): 552 """Installs the given control file. 553 554 The given file will be copied into the container. 555 556 @param control_file: Path to the control file to install. 557 """ 558 dst = os.path.join(constants.CONTROL_TEMP_PATH, 559 os.path.basename(control_file)) 560 self.copy(control_file, dst) 561 562 563 def copy(self, host_path, container_path): 564 """Copies files into the container. 565 566 @param host_path: Path to the source file/dir to be copied. 567 @param container_path: Path to the destination dir (in the container). 568 """ 569 dst_path = os.path.join(self.rootfs, 570 container_path.lstrip(os.path.sep)) 571 self._do_copy(src=host_path, dst=dst_path) 572 573 574 @property 575 def id(self): 576 """Returns the container ID.""" 577 return self._id 578 579 580 @id.setter 581 def id(self, new_id): 582 """Sets the container ID.""" 583 self._id = new_id; 584 # Persist the ID so other container objects can pick it up. 585 self._id.save(os.path.join(self.container_path, self.name)) 586 587 588 def _do_copy(self, src, dst): 589 """Copies files and directories on the host system. 590 591 @param src: The source file or directory. 592 @param dst: The destination file or directory. If the path to the 593 destination does not exist, it will be created. 594 """ 595 # Create the dst dir. mkdir -p will not fail if dst_dir exists. 596 dst_dir = os.path.dirname(dst) 597 # Make sure the source ends with `/.` if it's a directory. Otherwise 598 # command cp will not work. 599 if os.path.isdir(src) and os.path.split(src)[1] != '.': 600 src = os.path.join(src, '.') 601 utils.run("sudo sh -c 'mkdir -p \"%s\" && cp -RL \"%s\" \"%s\"'" % 602 (dst_dir, src, dst)) 603 604 def _set_lxc_config(self, key, value): 605 """Sets an LXC config value for this container. 606 607 Configuration changes made while a container is running don't take 608 effect until the container is restarted. Since this isn't a scenario 609 that should ever come up in our use cases, calling this method on a 610 running container will cause a ContainerError. 611 612 @param key: The LXC config key to set. 613 @param value: The value to use for the given key. 614 615 @raise error.ContainerError: If the container is already started. 616 """ 617 if self.is_running(): 618 raise error.ContainerError( 619 '_set_lxc_config(%s, %s) called on a running container.' % 620 (key, value)) 621 config_file = os.path.join(self.container_path, self.name, 'config') 622 config = '%s = %s' % (key, value) 623 utils.run( 624 constants.APPEND_CMD_FMT % {'content': config, 'file': config_file}) 625 626 627 def _get_lxc_config(self, key): 628 """Retrieves an LXC config value from the container. 629 630 @param key The key of the config value to retrieve. 631 """ 632 cmd = ('sudo lxc-info -P %s -n %s -c %s' % 633 (self.container_path, self.name, key)) 634 config = utils.run(cmd).stdout.strip().splitlines() 635 636 # Strip the decoration from line 1 of the output. 637 match = re.match('%s = (.*)' % key, config[0]) 638 if not match: 639 raise error.ContainerError( 640 'Config %s not found for container %s. (%s)' % 641 (key, self.name, ','.join(config))) 642 config[0] = match.group(1) 643 return config 644