1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import collections 6import json 7import logging 8import os 9import re 10import tempfile 11import time 12 13import common 14from autotest_lib.client.bin import utils 15from autotest_lib.client.common_lib import error 16from autotest_lib.site_utils.lxc import constants 17from autotest_lib.site_utils.lxc import lxc 18from autotest_lib.site_utils.lxc import utils as lxc_utils 19 20try: 21 from chromite.lib import metrics 22except ImportError: 23 metrics = utils.metrics_mock 24 25 26# Naming convention of test container, e.g., test_300_1422862512_2424, where: 27# 300: The test job ID. 28# 1422862512: The tick when container is created. 29# 2424: The PID of autoserv that starts the container. 30_TEST_CONTAINER_NAME_FMT = 'test_%s_%d_%d' 31# Name of the container ID file. 32_CONTAINER_ID_FILENAME = 'container_id.json' 33 34 35class ContainerId(collections.namedtuple('ContainerId', 36 ['job_id', 'creation_time', 'pid'])): 37 """An identifier for containers.""" 38 39 # Optimization. Avoids __dict__ creation. Empty because this subclass has 40 # no instance vars of its own. 41 __slots__ = () 42 43 44 def __str__(self): 45 return _TEST_CONTAINER_NAME_FMT % self 46 47 48 def save(self, path): 49 """Saves the ID to the given path. 50 51 @param path: Path to a directory where the container ID will be 52 serialized. 53 """ 54 dst = os.path.join(path, _CONTAINER_ID_FILENAME) 55 with open(dst, 'w') as f: 56 json.dump(self, f) 57 58 @classmethod 59 def load(cls, path): 60 """Reads the ID from the given path. 61 62 @param path: Path to check for a serialized container ID. 63 64 @return: A container ID if one is found on the given path, or None 65 otherwise. 66 67 @raise ValueError: If a JSON load error occurred. 68 @raise TypeError: If the file was valid JSON but didn't contain a valid 69 ContainerId. 70 """ 71 src = os.path.join(path, _CONTAINER_ID_FILENAME) 72 73 try: 74 with open(src, 'r') as f: 75 return cls(*json.load(f)) 76 except IOError: 77 # File not found, or couldn't be opened for some other reason. 78 # Treat all these cases as no ID. 79 return None 80 81 82 @classmethod 83 def create(cls, job_id, ctime=None, pid=None): 84 """Creates a new container ID. 85 86 @param job_id: The first field in the ID. 87 @param ctime: The second field in the ID. Optional. If not provided, 88 the current epoch timestamp is used. 89 @param pid: The third field in the ID. Optional. If not provided, the 90 PID of the current process is used. 91 """ 92 if ctime is None: 93 ctime = int(time.time()) 94 if pid is None: 95 pid = os.getpid() 96 return cls(job_id, ctime, pid) 97 98 99class Container(object): 100 """A wrapper class of an LXC container. 101 102 The wrapper class provides methods to interact with a container, e.g., 103 start, stop, destroy, run a command. It also has attributes of the 104 container, including: 105 name: Name of the container. 106 state: State of the container, e.g., ABORTING, RUNNING, STARTING, STOPPED, 107 or STOPPING. 108 109 lxc-ls can also collect other attributes of a container including: 110 ipv4: IP address for IPv4. 111 ipv6: IP address for IPv6. 112 autostart: If the container will autostart at system boot. 113 pid: Process ID of the container. 114 memory: Memory used by the container, as a string, e.g., "6.2MB" 115 ram: Physical ram used by the container, as a string, e.g., "6.2MB" 116 swap: swap used by the container, as a string, e.g., "1.0MB" 117 118 For performance reason, such info is not collected for now. 119 120 The attributes available are defined in ATTRIBUTES constant. 121 """ 122 123 def __init__(self, container_path, name, attribute_values, src=None, 124 snapshot=False): 125 """Initialize an object of LXC container with given attribute values. 126 127 @param container_path: Directory that stores the container. 128 @param name: Name of the container. 129 @param attribute_values: A dictionary of attribute values for the 130 container. 131 @param src: An optional source container. If provided, the source 132 continer is cloned, and the new container will point to the 133 clone. 134 @param snapshot: If a source container was specified, this argument 135 specifies whether or not to create a snapshot clone. 136 The default is to attempt to create a snapshot. 137 If a snapshot is requested and creating the snapshot 138 fails, a full clone will be attempted. 139 """ 140 self.container_path = os.path.realpath(container_path) 141 # Path to the rootfs of the container. This will be initialized when 142 # property rootfs is retrieved. 143 self._rootfs = None 144 self.name = name 145 for attribute, value in attribute_values.iteritems(): 146 setattr(self, attribute, value) 147 148 # Clone the container 149 if src is not None: 150 # Clone the source container to initialize this one. 151 lxc_utils.clone(src.container_path, src.name, self.container_path, 152 self.name, snapshot) 153 # Newly cloned containers have no ID. 154 self._id = None 155 else: 156 # This may be an existing container. Try to read the ID. 157 try: 158 self._id = ContainerId.load( 159 os.path.join(self.container_path, self.name)) 160 except (ValueError, TypeError): 161 # Ignore load errors. ContainerBucket currently queries every 162 # container quite frequently, and emitting exceptions here would 163 # cause any invalid containers on a server to block all 164 # ContainerBucket.get_all calls (see crbug/783865). 165 # TODO(kenobi): Containers with invalid ID files are probably 166 # the result of an aborted or failed operation. There is a 167 # non-zero chance that such containers would contain leftover 168 # state, or themselves be corrupted or invalid. Should we 169 # provide APIs for checking if a container is in this state? 170 logging.exception('Error loading ID for container %s:', 171 self.name) 172 self._id = None 173 174 175 @classmethod 176 def create_from_existing_dir(cls, lxc_path, name, **kwargs): 177 """Creates a new container instance for an lxc container that already 178 exists on disk. 179 180 @param lxc_path: The LXC path for the container. 181 @param name: The container name. 182 183 @raise error.ContainerError: If the container doesn't already exist. 184 185 @return: The new container. 186 """ 187 return cls(lxc_path, name, kwargs) 188 189 190 # Containers have a name and an ID. The name is simply the name of the LXC 191 # container. The ID is the actual key that is used to identify the 192 # container to the autoserv system. In the case of a JIT-created container, 193 # we have the ID at the container's creation time so we use that to name the 194 # container. This may not be the case for other types of containers. 195 @classmethod 196 def clone(cls, src, new_name=None, new_path=None, snapshot=False, 197 cleanup=False): 198 """Creates a clone of this container. 199 200 @param src: The original container. 201 @param new_name: Name for the cloned container. If this is not 202 provided, a random unique container name will be 203 generated. 204 @param new_path: LXC path for the cloned container (optional; if not 205 specified, the new container is created in the same 206 directory as the source container). 207 @param snapshot: Whether to snapshot, or create a full clone. Note that 208 snapshot cloning is not supported on all platforms. If 209 this code is running on a platform that does not 210 support snapshot clones, this flag is ignored. 211 @param cleanup: If a container with the given name and path already 212 exist, clean it up first. 213 """ 214 if new_path is None: 215 new_path = src.container_path 216 217 if new_name is None: 218 _, new_name = os.path.split( 219 tempfile.mkdtemp(dir=new_path, prefix='container.')) 220 logging.debug('Generating new name for container: %s', new_name) 221 else: 222 # If a container exists at this location, clean it up first 223 container_folder = os.path.join(new_path, new_name) 224 if lxc_utils.path_exists(container_folder): 225 if not cleanup: 226 raise error.ContainerError('Container %s already exists.' % 227 new_name) 228 container = Container.create_from_existing_dir(new_path, 229 new_name) 230 try: 231 container.destroy() 232 except error.CmdError as e: 233 # The container could be created in a incompleted 234 # state. Delete the container folder instead. 235 logging.warn('Failed to destroy container %s, error: %s', 236 new_name, e) 237 utils.run('sudo rm -rf "%s"' % container_folder) 238 # Create the directory prior to creating the new container. This 239 # puts the ownership of the container under the current process's 240 # user, rather than root. This is necessary to enable the 241 # ContainerId to serialize properly. 242 os.mkdir(container_folder) 243 244 # Create and return the new container. 245 new_container = cls(new_path, new_name, {}, src, snapshot) 246 247 return new_container 248 249 250 def refresh_status(self): 251 """Refresh the status information of the container. 252 """ 253 containers = lxc.get_container_info(self.container_path, name=self.name) 254 if not containers: 255 raise error.ContainerError( 256 'No container found in directory %s with name of %s.' % 257 (self.container_path, self.name)) 258 attribute_values = containers[0] 259 for attribute, value in attribute_values.iteritems(): 260 setattr(self, attribute, value) 261 262 263 @property 264 def rootfs(self): 265 """Path to the rootfs of the container. 266 267 This property returns the path to the rootfs of the container, that is, 268 the folder where the container stores its local files. It reads the 269 attribute lxc.rootfs from the config file of the container, e.g., 270 lxc.rootfs = /usr/local/autotest/containers/t4/rootfs 271 If the container is created with snapshot, the rootfs is a chain of 272 folders, separated by `:` and ordered by how the snapshot is created, 273 e.g., 274 lxc.rootfs = overlayfs:/usr/local/autotest/containers/base/rootfs: 275 /usr/local/autotest/containers/t4_s/delta0 276 This function returns the last folder in the chain, in above example, 277 that is `/usr/local/autotest/containers/t4_s/delta0` 278 279 Files in the rootfs will be accessible directly within container. For 280 example, a folder in host "[rootfs]/usr/local/file1", can be accessed 281 inside container by path "/usr/local/file1". Note that symlink in the 282 host can not across host/container boundary, instead, directory mount 283 should be used, refer to function mount_dir. 284 285 @return: Path to the rootfs of the container. 286 """ 287 if not self._rootfs: 288 lxc_rootfs = self._get_lxc_config('lxc.rootfs')[0] 289 cloned_from_snapshot = ':' in lxc_rootfs 290 if cloned_from_snapshot: 291 self._rootfs = lxc_rootfs.split(':')[-1] 292 else: 293 self._rootfs = lxc_rootfs 294 return self._rootfs 295 296 297 def attach_run(self, command, bash=True): 298 """Attach to a given container and run the given command. 299 300 @param command: Command to run in the container. 301 @param bash: Run the command through bash -c "command". This allows 302 pipes to be used in command. Default is set to True. 303 304 @return: The output of the command. 305 306 @raise error.CmdError: If container does not exist, or not running. 307 """ 308 cmd = 'sudo lxc-attach -P %s -n %s' % (self.container_path, self.name) 309 if bash and not command.startswith('bash -c'): 310 command = 'bash -c "%s"' % utils.sh_escape(command) 311 cmd += ' -- %s' % command 312 # TODO(dshi): crbug.com/459344 Set sudo to default to False when test 313 # container can be unprivileged container. 314 return utils.run(cmd) 315 316 317 def is_network_up(self): 318 """Check if network is up in the container by curl base container url. 319 320 @return: True if the network is up, otherwise False. 321 """ 322 try: 323 self.attach_run('curl --head %s' % constants.CONTAINER_BASE_URL) 324 return True 325 except error.CmdError as e: 326 logging.debug(e) 327 return False 328 329 330 @metrics.SecondsTimerDecorator( 331 '%s/container_start_duration' % constants.STATS_KEY) 332 def start(self, wait_for_network=True): 333 """Start the container. 334 335 @param wait_for_network: True to wait for network to be up. Default is 336 set to True. 337 338 @raise ContainerError: If container does not exist, or fails to start. 339 """ 340 cmd = 'sudo lxc-start -P %s -n %s -d' % (self.container_path, self.name) 341 output = utils.run(cmd).stdout 342 if not self.is_running(): 343 raise error.ContainerError( 344 'Container %s failed to start. lxc command output:\n%s' % 345 (os.path.join(self.container_path, self.name), 346 output)) 347 348 if wait_for_network: 349 logging.debug('Wait for network to be up.') 350 start_time = time.time() 351 utils.poll_for_condition( 352 condition=self.is_network_up, 353 timeout=constants.NETWORK_INIT_TIMEOUT, 354 sleep_interval=constants.NETWORK_INIT_CHECK_INTERVAL) 355 logging.debug('Network is up after %.2f seconds.', 356 time.time() - start_time) 357 358 359 @metrics.SecondsTimerDecorator( 360 '%s/container_stop_duration' % constants.STATS_KEY) 361 def stop(self): 362 """Stop the container. 363 364 @raise ContainerError: If container does not exist, or fails to start. 365 """ 366 cmd = 'sudo lxc-stop -P %s -n %s' % (self.container_path, self.name) 367 output = utils.run(cmd).stdout 368 self.refresh_status() 369 if self.state != 'STOPPED': 370 raise error.ContainerError( 371 'Container %s failed to be stopped. lxc command output:\n' 372 '%s' % (os.path.join(self.container_path, self.name), 373 output)) 374 375 376 @metrics.SecondsTimerDecorator( 377 '%s/container_destroy_duration' % constants.STATS_KEY) 378 def destroy(self, force=True): 379 """Destroy the container. 380 381 @param force: Set to True to force to destroy the container even if it's 382 running. This is faster than stop a container first then 383 try to destroy it. Default is set to True. 384 385 @raise ContainerError: If container does not exist or failed to destroy 386 the container. 387 """ 388 logging.debug('Destroying container %s/%s', 389 self.container_path, 390 self.name) 391 cmd = 'sudo lxc-destroy -P %s -n %s' % (self.container_path, 392 self.name) 393 if force: 394 cmd += ' -f' 395 utils.run(cmd) 396 397 398 def mount_dir(self, source, destination, readonly=False): 399 """Mount a directory in host to a directory in the container. 400 401 @param source: Directory in host to be mounted. 402 @param destination: Directory in container to mount the source directory 403 @param readonly: Set to True to make a readonly mount, default is False. 404 """ 405 # Destination path in container must be relative. 406 destination = destination.lstrip('/') 407 # Create directory in container for mount. Changes to container rootfs 408 # require sudo. 409 utils.run('sudo mkdir -p %s' % os.path.join(self.rootfs, destination)) 410 mount = ('%s %s none bind%s 0 0' % 411 (source, destination, ',ro' if readonly else '')) 412 self._set_lxc_config('lxc.mount.entry', mount) 413 414 def verify_autotest_setup(self, job_folder): 415 """Verify autotest code is set up properly in the container. 416 417 @param job_folder: Name of the job result folder. 418 419 @raise ContainerError: If autotest code is not set up properly. 420 """ 421 # Test autotest code is setup by verifying a list of 422 # (directory, minimum file count) 423 directories_to_check = [ 424 (constants.CONTAINER_AUTOTEST_DIR, 3), 425 (constants.RESULT_DIR_FMT % job_folder, 0), 426 (constants.CONTAINER_SITE_PACKAGES_PATH, 3)] 427 for directory, count in directories_to_check: 428 result = self.attach_run(command=(constants.COUNT_FILE_CMD % 429 {'dir': directory})).stdout 430 logging.debug('%s entries in %s.', int(result), directory) 431 if int(result) < count: 432 raise error.ContainerError('%s is not properly set up.' % 433 directory) 434 # lxc-attach and run command does not run in shell, thus .bashrc is not 435 # loaded. Following command creates a symlink in /usr/bin/ for gsutil 436 # if it's installed. 437 # TODO(dshi): Remove this code after lab container is updated with 438 # gsutil installed in /usr/bin/ 439 self.attach_run('test -f /root/gsutil/gsutil && ' 440 'ln -s /root/gsutil/gsutil /usr/bin/gsutil || true') 441 442 443 def modify_import_order(self): 444 """Swap the python import order of lib and local/lib. 445 446 In Moblab, the host's python modules located in 447 /usr/lib64/python2.7/site-packages is mounted to following folder inside 448 container: /usr/local/lib/python2.7/dist-packages/. The modules include 449 an old version of requests module, which is used in autotest 450 site-packages. For test, the module is only used in 451 dev_server/symbolicate_dump for requests.call and requests.codes.OK. 452 When pip is installed inside the container, it installs requests module 453 with version of 2.2.1 in /usr/lib/python2.7/dist-packages/. The version 454 is newer than the one used in autotest site-packages, but not the latest 455 either. 456 According to /usr/lib/python2.7/site.py, modules in /usr/local/lib are 457 imported before the ones in /usr/lib. That leads to pip to use the older 458 version of requests (0.11.2), and it will fail. On the other hand, 459 requests module 2.2.1 can't be installed in CrOS (refer to CL:265759), 460 and higher version of requests module can't work with pip. 461 The only fix to resolve this is to switch the import order, so modules 462 in /usr/lib can be imported before /usr/local/lib. 463 """ 464 site_module = '/usr/lib/python2.7/site.py' 465 self.attach_run("sed -i ':a;N;$!ba;s/\"local\/lib\",\\n/" 466 "\"lib_placeholder\",\\n/g' %s" % site_module) 467 self.attach_run("sed -i ':a;N;$!ba;s/\"lib\",\\n/" 468 "\"local\/lib\",\\n/g' %s" % site_module) 469 self.attach_run('sed -i "s/lib_placeholder/lib/g" %s' % 470 site_module) 471 472 473 def is_running(self): 474 """Returns whether or not this container is currently running.""" 475 self.refresh_status() 476 return self.state == 'RUNNING' 477 478 479 def set_hostname(self, hostname): 480 """Sets the hostname within the container. 481 482 This method can only be called on a running container. 483 484 @param hostname The new container hostname. 485 486 @raise ContainerError: If the container is not running. 487 """ 488 if not self.is_running(): 489 raise error.ContainerError( 490 'set_hostname can only be called on running containers.') 491 492 self.attach_run('hostname %s' % (hostname)) 493 self.attach_run(constants.APPEND_CMD_FMT % { 494 'content': '127.0.0.1 %s' % (hostname), 495 'file': '/etc/hosts'}) 496 497 498 def install_ssp(self, ssp_url): 499 """Downloads and installs the given server package. 500 501 @param ssp_url: The URL of the ssp to download and install. 502 """ 503 usr_local_path = os.path.join(self.rootfs, 'usr', 'local') 504 autotest_pkg_path = os.path.join(usr_local_path, 505 'autotest_server_package.tar.bz2') 506 # Changes within the container rootfs require sudo. 507 utils.run('sudo mkdir -p %s'% usr_local_path) 508 509 lxc.download_extract(ssp_url, autotest_pkg_path, usr_local_path) 510 511 512 def install_control_file(self, control_file): 513 """Installs the given control file. 514 515 The given file will be copied into the container. 516 517 @param control_file: Path to the control file to install. 518 """ 519 dst = os.path.join(constants.CONTROL_TEMP_PATH, 520 os.path.basename(control_file)) 521 self.copy(control_file, dst) 522 523 524 def copy(self, host_path, container_path): 525 """Copies files into the container. 526 527 @param host_path: Path to the source file/dir to be copied. 528 @param container_path: Path to the destination dir (in the container). 529 """ 530 dst_path = os.path.join(self.rootfs, 531 container_path.lstrip(os.path.sep)) 532 self._do_copy(src=host_path, dst=dst_path) 533 534 535 @property 536 def id(self): 537 """Returns the container ID.""" 538 return self._id 539 540 541 @id.setter 542 def id(self, new_id): 543 """Sets the container ID.""" 544 self._id = new_id; 545 # Persist the ID so other container objects can pick it up. 546 self._id.save(os.path.join(self.container_path, self.name)) 547 548 549 def _do_copy(self, src, dst): 550 """Copies files and directories on the host system. 551 552 @param src: The source file or directory. 553 @param dst: The destination file or directory. If the path to the 554 destination does not exist, it will be created. 555 """ 556 # Create the dst dir. mkdir -p will not fail if dst_dir exists. 557 dst_dir = os.path.dirname(dst) 558 # Make sure the source ends with `/.` if it's a directory. Otherwise 559 # command cp will not work. 560 if os.path.isdir(src) and os.path.split(src)[1] != '.': 561 src = os.path.join(src, '.') 562 utils.run("sudo sh -c 'mkdir -p \"%s\" && cp -RL \"%s\" \"%s\"'" % 563 (dst_dir, src, dst)) 564 565 def _set_lxc_config(self, key, value): 566 """Sets an LXC config value for this container. 567 568 Configuration changes made while a container is running don't take 569 effect until the container is restarted. Since this isn't a scenario 570 that should ever come up in our use cases, calling this method on a 571 running container will cause a ContainerError. 572 573 @param key: The LXC config key to set. 574 @param value: The value to use for the given key. 575 576 @raise error.ContainerError: If the container is already started. 577 """ 578 if self.is_running(): 579 raise error.ContainerError( 580 '_set_lxc_config(%s, %s) called on a running container.' % 581 (key, value)) 582 config_file = os.path.join(self.container_path, self.name, 'config') 583 config = '%s = %s' % (key, value) 584 utils.run( 585 constants.APPEND_CMD_FMT % {'content': config, 'file': config_file}) 586 587 588 def _get_lxc_config(self, key): 589 """Retrieves an LXC config value from the container. 590 591 @param key The key of the config value to retrieve. 592 """ 593 cmd = ('sudo lxc-info -P %s -n %s -c %s' % 594 (self.container_path, self.name, key)) 595 config = utils.run(cmd).stdout.strip().splitlines() 596 597 # Strip the decoration from line 1 of the output. 598 match = re.match('%s = (.*)' % key, config[0]) 599 if not match: 600 raise error.ContainerError( 601 'Config %s not found for container %s. (%s)' % 602 (key, self.name, ','.join(config))) 603 config[0] = match.group(1) 604 return config 605