1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import abc 6import logging 7import os 8import re 9 10import common 11from autotest_lib.client.common_lib import error, utils 12from autotest_lib.client.common_lib.cros import dev_server 13 14 15# Relevant CrosDynamicSuiteExceptions are defined in client/common_lib/error.py. 16 17 18class ControlFileGetter(object): 19 """ 20 Interface for classes that can list and fetch known control files. 21 """ 22 23 __metaclass__ = abc.ABCMeta 24 25 26 @abc.abstractmethod 27 def get_control_file_list(self, suite_name=''): 28 """ 29 Gather a list of paths to control files. 30 31 @param suite_name: The name of a suite we would like control files for. 32 @return A list of file paths. 33 @throws NoControlFileList if there is an error while listing. 34 """ 35 pass 36 37 38 @abc.abstractmethod 39 def get_control_file_contents(self, test_path): 40 """ 41 Given a path to a control file, return its contents. 42 43 @param test_path: the path to the control file. 44 @return the contents of the control file specified by the path. 45 @throws ControlFileNotFound if the file cannot be retrieved. 46 """ 47 pass 48 49 50 @abc.abstractmethod 51 def get_control_file_contents_by_name(self, test_name): 52 """ 53 Given the name of a control file, return its contents. 54 55 @param test_name: the name of the test whose control file is desired. 56 @return the contents of the control file specified by the name. 57 @throws ControlFileNotFound if the file cannot be retrieved. 58 """ 59 pass 60 61 62class SuiteControlFileGetter(ControlFileGetter): 63 """Interface that additionally supports getting by suite.""" 64 65 66 @abc.abstractmethod 67 def get_suite_info(self, suite_name=''): 68 """ 69 Gather the control paths and contents of all the control files. 70 71 @param suite_name: The name of a suite we would like control files for. 72 @return the control paths and contents of all the control files 73 specified by the name. 74 @throws SuiteControlFileException if the info cannot be retrieved. 75 """ 76 pass 77 78 79class CacheingAndFilteringControlFileGetter(ControlFileGetter): 80 """Wraps ControlFileGetter to cache the retrieved control file list and 81 filter out unwanted control files.""" 82 83 CONTROL_FILE_FILTERS = ['src/debian/control'] 84 85 def __init__(self): 86 super(CacheingAndFilteringControlFileGetter, self).__init__() 87 self._files = [] 88 89 90 def get_control_file_list(self, suite_name=''): 91 """ 92 Gather a list of paths to control files. 93 94 Gets a list of control files; populates |self._files| with that list 95 and then returns the paths to all useful and wanted files in the list. 96 97 @param suite_name: The name of a suite we would like control files for. 98 @return A list of file paths. 99 @throws NoControlFileList if there is an error while listing. 100 """ 101 files = self._get_control_file_list(suite_name=suite_name) 102 for cf_filter in self.CONTROL_FILE_FILTERS: 103 files = filter(lambda path: not path.endswith(cf_filter), files) 104 self._files = files 105 return self._files 106 107 108 @abc.abstractmethod 109 def _get_control_file_list(self, suite_name=''): 110 pass 111 112 113 def get_control_file_path(self, test_name): 114 """ 115 Given the name of a control file, return its path. 116 117 Searches through previously-compiled list in |self._files| for a 118 test named |test_name| and returns the contents of the control file 119 for that test if it is found. 120 121 @param test_name: the name of the test whose control file is desired. 122 @return control file path 123 @throws ControlFileNotFound if the file cannot be retrieved. 124 """ 125 if not self._files and not self.get_control_file_list(): 126 raise error.ControlFileNotFound('No control files found.') 127 128 if 'control' not in test_name: 129 regexp = re.compile(os.path.join(test_name, 'control$')) 130 else: 131 regexp = re.compile(test_name + '$') 132 candidates = filter(regexp.search, self._files) 133 if not candidates: 134 raise error.ControlFileNotFound('No control file for ' + test_name) 135 if len(candidates) > 1: 136 raise error.ControlFileNotFound(test_name + ' is not unique.') 137 return candidates[0] 138 139 140 def get_control_file_contents_by_name(self, test_name): 141 """ 142 Given the name of a control file, return its contents. 143 144 Searches through previously-compiled list in |self._files| for a 145 test named |test_name| and returns the contents of the control file 146 for that test if it is found. 147 148 @param test_name: the name of the test whose control file is desired. 149 @return the contents of the control file specified by the name. 150 @throws ControlFileNotFound if the file cannot be retrieved. 151 """ 152 path = self.get_control_file_path(test_name) 153 return self.get_control_file_contents(path) 154 155 156class FileSystemGetter(CacheingAndFilteringControlFileGetter): 157 """ 158 Class that can list and fetch known control files from disk. 159 160 @var _CONTROL_PATTERN: control file name format to match. 161 """ 162 163 _CONTROL_PATTERN = '^control(?:\..+)?$' 164 165 def __init__(self, paths): 166 """ 167 @param paths: base directories to start search. 168 """ 169 super(FileSystemGetter, self).__init__() 170 self._paths = paths 171 172 173 def _is_useful_file(self, name): 174 return '__init__.py' not in name and '.svn' not in name 175 176 177 def _get_control_file_list(self, suite_name=''): 178 """ 179 Gather a list of paths to control files under |self._paths|. 180 181 Searches under |self._paths| for files that match 182 |self._CONTROL_PATTERN|. Populates |self._files| with that list 183 and then returns the paths to all useful files in the list. 184 185 @param suite_name: The name of a suite we would like control files for. 186 @return A list of files that match |self._CONTROL_PATTERN|. 187 @throws NoControlFileList if we find no files. 188 """ 189 if suite_name: 190 logging.debug('Getting control files for a specific suite has ' 191 'not been implemented for FileSystemGetter. ' 192 'Getting all control files instead.') 193 194 195 regexp = re.compile(self._CONTROL_PATTERN) 196 directories = self._paths 197 # Do not explore site-packages. (crbug.com/771823) 198 # Do not explore venv. (b/67416549) 199 # (Do not pass Go. Do not collect $200.) 200 blacklist = {'site-packages', 'venv'} 201 while len(directories) > 0: 202 directory = directories.pop() 203 if not os.path.exists(directory): 204 continue 205 # TODO(crbug.com/771827): This traverses everything, 206 # including results and containers. Make it stop doing that. 207 try: 208 for name in os.listdir(directory): 209 if name in blacklist: 210 continue 211 fullpath = os.path.join(directory, name) 212 if os.path.isfile(fullpath): 213 if regexp.search(name): 214 # if we are a control file 215 self._files.append(fullpath) 216 elif os.path.isdir(fullpath): 217 directories.append(fullpath) 218 except OSError: 219 # Some directories under results/ like the Chrome Crash 220 # Reports will cause issues when attempted to be searched. 221 logging.error('Unable to search directory %s for control ' 222 'files.', directory) 223 pass 224 if not self._files: 225 msg = 'No control files under ' + ','.join(self._paths) 226 raise error.NoControlFileList(msg) 227 return [f for f in self._files if self._is_useful_file(f)] 228 229 230 def get_control_file_contents(self, test_path): 231 """ 232 Get the contents of the control file at |test_path|. 233 234 @return The contents of the aforementioned file. 235 @throws ControlFileNotFound if the file cannot be retrieved. 236 """ 237 try: 238 return utils.read_file(test_path) 239 except EnvironmentError as (errno, strerror): 240 msg = "Can't retrieve {0}: {1} ({2})".format(test_path, 241 strerror, 242 errno) 243 raise error.ControlFileNotFound(msg) 244 245 246class DevServerGetter(CacheingAndFilteringControlFileGetter, 247 SuiteControlFileGetter): 248 """Class that can list and fetch known control files from DevServer. 249 250 @var _CONTROL_PATTERN: control file name format to match. 251 """ 252 def __init__(self, build, ds): 253 """ 254 @param build: The build from which to get control files. 255 @param ds: An existing dev_server.DevServer object to use. 256 """ 257 super(DevServerGetter, self).__init__() 258 self._dev_server = ds 259 self._build = build 260 261 262 @staticmethod 263 def create(build, ds=None): 264 """Wraps constructor. Can be mocked for testing purposes. 265 @param build: The build from which to get control files. 266 @param ds: An existing dev_server.DevServer object to use 267 (default=None) 268 @returns: New DevServerGetter. 269 """ 270 return DevServerGetter(build, ds) 271 272 273 def _get_control_file_list(self, suite_name=''): 274 """ 275 Gather a list of paths to control files from |self._dev_server|. 276 277 Get a listing of all the control files for |self._build| on 278 |self._dev_server|. Populates |self._files| with that list 279 and then returns paths (under the autotest dir) to them. If suite_name 280 is specified, this method populates |self._files| with the control 281 files from just the specified suite. 282 283 @param suite_name: The name of a suite we would like control files for. 284 @return A list of control file paths. 285 @throws NoControlFileList if there is an error while listing. 286 """ 287 try: 288 return self._dev_server.list_control_files(self._build, 289 suite_name=suite_name) 290 except dev_server.DevServerException as e: 291 raise error.NoControlFileList(e) 292 293 294 def get_control_file_contents(self, test_path): 295 """ 296 Return the contents of |test_path| from |self._dev_server|. 297 298 Get the contents of the control file at |test_path| for |self._build| on 299 |self._dev_server|. 300 301 @return The contents of |test_path|. None on failure. 302 @throws ControlFileNotFound if the file cannot be retrieved. 303 """ 304 try: 305 return self._dev_server.get_control_file(self._build, test_path) 306 except dev_server.DevServerException as e: 307 raise error.ControlFileNotFound(e) 308 309 310 def _list_suite_controls(self, suite_name=''): 311 """ 312 Gather a dict {path:content} of all control files from 313 |self._dev_server|. 314 315 Get a dict of contents of all the control files for |self._build| on 316 |self._dev_server|: path is the key, and the control file content is 317 the value. 318 319 @param suite_name: The name of a suite we would like control files for. 320 @return A dict of paths and contents of all control files. 321 @throws NoControlFileList if there is an error while listing. 322 """ 323 try: 324 return self._dev_server.list_suite_controls(self._build, 325 suite_name=suite_name) 326 except dev_server.DevServerException as e: 327 raise error.SuiteControlFileException(e) 328 329 330 def get_suite_info(self, suite_name=''): 331 """ 332 Gather info of a list of control files from |self._dev_server|. 333 334 The info is a dict: {control_path: control_file_content} for 335 |self._build| on |self._dev_server|. 336 337 @param suite_name: The name of a suite we would like control files for. 338 @return A dict of paths and contents of all control files: 339 {path1: content1, path2: content2, ..., pathX: contentX} 340 """ 341 file_contents = self._list_suite_controls(suite_name=suite_name) 342 files = file_contents.keys() 343 for cf_filter in self.CONTROL_FILE_FILTERS: 344 files = filter(lambda path: not path.endswith(cf_filter), files) 345 self._files = files 346 return {f: file_contents[f] for f in files} 347