1# Please keep this code python 2.4 compatible and stand alone. 2 3import logging, os, shutil, sys, tempfile, time, urllib2 4import subprocess, re 5from distutils.version import LooseVersion 6 7from autotest_lib.client.common_lib import autotemp, revision_control, utils 8 9_READ_SIZE = 64*1024 10_MAX_PACKAGE_SIZE = 100*1024*1024 11 12 13class Error(Exception): 14 """Local exception to be raised by code in this file.""" 15 16class FetchError(Error): 17 """Failed to fetch a package from any of its listed URLs.""" 18 19 20def _checksum_file(full_path): 21 """@returns The hex checksum of a file given its pathname.""" 22 inputfile = open(full_path, 'rb') 23 try: 24 hex_sum = utils.hash('sha1', inputfile.read()).hexdigest() 25 finally: 26 inputfile.close() 27 return hex_sum 28 29 30def system(commandline): 31 """Same as os.system(commandline) but logs the command first. 32 33 @param commandline: commandline to be called. 34 """ 35 logging.info(commandline) 36 return os.system(commandline) 37 38 39def find_top_of_autotest_tree(): 40 """@returns The full path to the top of the autotest directory tree.""" 41 dirname = os.path.dirname(__file__) 42 autotest_dir = os.path.abspath(os.path.join(dirname, '..')) 43 return autotest_dir 44 45 46class ExternalPackage(object): 47 """ 48 Defines an external package with URLs to fetch its sources from and 49 a build_and_install() method to unpack it, build it and install it 50 beneath our own autotest/site-packages directory. 51 52 Base Class. Subclass this to define packages. 53 Note: Unless your subclass has a specific reason to, it should not 54 re-install the package every time build_externals is invoked, as this 55 happens periodically through the scheduler. To avoid doing so the is_needed 56 method needs to return an appropriate value. 57 58 Attributes: 59 @attribute urls - A tuple of URLs to try fetching the package from. 60 @attribute local_filename - A local filename to use when saving the 61 fetched package. 62 @attribute dist_name - The name of the Python distribution. For example, 63 the package MySQLdb is included in the distribution named 64 MySQL-python. This is generally the PyPI name. Defaults to the 65 name part of the local_filename. 66 @attribute hex_sum - The hex digest (currently SHA1) of this package 67 to be used to verify its contents. 68 @attribute module_name - The installed python module name to be used for 69 for a version check. Defaults to the lower case class name with 70 the word Package stripped off. 71 @attribute extracted_package_path - The path to package directory after 72 extracting. 73 @attribute version - The desired minimum package version. 74 @attribute os_requirements - A dictionary mapping pathname tuples on the 75 the OS distribution to a likely name of a package the user 76 needs to install on their system in order to get this file. 77 One of the files in the tuple must exist. 78 @attribute name - Read only, the printable name of the package. 79 @attribute subclasses - This class attribute holds a list of all defined 80 subclasses. It is constructed dynamically using the metaclass. 81 """ 82 # Modules that are meant to be installed in system directory, rather than 83 # autotest/site-packages. These modules should be skipped if the module 84 # is already installed in system directory. This prevents an older version 85 # of the module from being installed in system directory. 86 SYSTEM_MODULES = ['setuptools'] 87 88 subclasses = [] 89 urls = () 90 local_filename = None 91 dist_name = None 92 hex_sum = None 93 module_name = None 94 version = None 95 os_requirements = None 96 97 98 class __metaclass__(type): 99 """Any time a subclass is defined, add it to our list.""" 100 def __init__(mcs, name, bases, dict): 101 if name != 'ExternalPackage' and not name.startswith('_'): 102 mcs.subclasses.append(mcs) 103 104 105 def __init__(self): 106 self.verified_package = '' 107 if not self.module_name: 108 self.module_name = self.name.lower() 109 if not self.dist_name and self.local_filename: 110 self.dist_name = self.local_filename[:self.local_filename.rindex('-')] 111 self.installed_version = '' 112 113 114 @property 115 def extracted_package_path(self): 116 """Return the package path after extracting. 117 118 If the package has assigned its own extracted_package_path, use it. 119 Or use part of its local_filename as the extracting path. 120 """ 121 return self.local_filename[:-len(self._get_extension( 122 self.local_filename))] 123 124 125 @property 126 def name(self): 127 """Return the class name with any trailing 'Package' stripped off.""" 128 class_name = self.__class__.__name__ 129 if class_name.endswith('Package'): 130 return class_name[:-len('Package')] 131 return class_name 132 133 134 def is_needed(self, install_dir): 135 """ 136 Check to see if we need to reinstall a package. This is contingent on: 137 1. Module name: If the name of the module is different from the package, 138 the class that installs it needs to specify a module_name string, 139 so we can try importing the module. 140 141 2. Installed version: If the module doesn't contain a __version__ the 142 class that installs it needs to override the 143 _get_installed_version_from_module method to return an appropriate 144 version string. 145 146 3. Version/Minimum version: The class that installs the package should 147 contain a version string, and an optional minimum version string. 148 149 4. install_dir: If the module exists in a different directory, e.g., 150 /usr/lib/python2.7/dist-packages/, the module will be forced to be 151 installed in install_dir. 152 153 @param install_dir: install directory. 154 @returns True if self.module_name needs to be built and installed. 155 """ 156 if not self.module_name or not self.version: 157 logging.warning('version and module_name required for ' 158 'is_needed() check to work.') 159 return True 160 try: 161 module = __import__(self.module_name) 162 except ImportError, e: 163 logging.info("%s isn't present. Will install.", self.module_name) 164 return True 165 if (not module.__file__.startswith(install_dir) and 166 not self.module_name in self.SYSTEM_MODULES): 167 logging.info('Module %s is installed in %s, rather than %s. The ' 168 'module will be forced to be installed in %s.', 169 self.module_name, module.__file__, install_dir, 170 install_dir) 171 return True 172 self.installed_version = self._get_installed_version_from_module(module) 173 if not self.installed_version: 174 return True 175 176 logging.info('imported %s version %s.', self.module_name, 177 self.installed_version) 178 if hasattr(self, 'minimum_version'): 179 return LooseVersion(self.minimum_version) > LooseVersion( 180 self.installed_version) 181 else: 182 return LooseVersion(self.version) > LooseVersion( 183 self.installed_version) 184 185 186 def _get_installed_version_from_module(self, module): 187 """Ask our module its version string and return it or '' if unknown.""" 188 try: 189 return module.__version__ 190 except AttributeError: 191 logging.error('could not get version from %s', module) 192 return '' 193 194 195 def _build_and_install(self, install_dir): 196 """Subclasses MUST provide their own implementation.""" 197 raise NotImplementedError 198 199 200 def _build_and_install_current_dir(self, install_dir): 201 """ 202 Subclasses that use _build_and_install_from_package() MUST provide 203 their own implementation of this method. 204 """ 205 raise NotImplementedError 206 207 208 def build_and_install(self, install_dir): 209 """ 210 Builds and installs the package. It must have been fetched already. 211 212 @param install_dir - The package installation directory. If it does 213 not exist it will be created. 214 """ 215 if not self.verified_package: 216 raise Error('Must call fetch() first. - %s' % self.name) 217 self._check_os_requirements() 218 return self._build_and_install(install_dir) 219 220 221 def _check_os_requirements(self): 222 if not self.os_requirements: 223 return 224 failed = False 225 for file_names, package_name in self.os_requirements.iteritems(): 226 if not any(os.path.exists(file_name) for file_name in file_names): 227 failed = True 228 logging.error('Can\'t find %s, %s probably needs it.', 229 ' or '.join(file_names), self.name) 230 logging.error('Perhaps you need to install something similar ' 231 'to the %s package for OS first.', package_name) 232 if failed: 233 raise Error('Missing OS requirements for %s. (see above)' % 234 self.name) 235 236 237 def _build_and_install_current_dir_setup_py(self, install_dir): 238 """For use as a _build_and_install_current_dir implementation.""" 239 egg_path = self._build_egg_using_setup_py(setup_py='setup.py') 240 if not egg_path: 241 return False 242 return self._install_from_egg(install_dir, egg_path) 243 244 245 def _build_and_install_current_dir_setupegg_py(self, install_dir): 246 """For use as a _build_and_install_current_dir implementation.""" 247 egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py') 248 if not egg_path: 249 return False 250 return self._install_from_egg(install_dir, egg_path) 251 252 253 def _build_and_install_current_dir_noegg(self, install_dir): 254 if not self._build_using_setup_py(): 255 return False 256 return self._install_using_setup_py_and_rsync(install_dir) 257 258 259 def _get_extension(self, package): 260 """Get extension of package.""" 261 valid_package_extensions = ['.tar.gz', '.tar.bz2', '.zip'] 262 extension = None 263 264 for ext in valid_package_extensions: 265 if package.endswith(ext): 266 extension = ext 267 break 268 269 if not extension: 270 raise Error('Unexpected package file extension on %s' % package) 271 272 return extension 273 274 275 def _build_and_install_from_package(self, install_dir): 276 """ 277 This method may be used as a _build_and_install() implementation 278 for subclasses if they implement _build_and_install_current_dir(). 279 280 Extracts the .tar.gz file, chdirs into the extracted directory 281 (which is assumed to match the tar filename) and calls 282 _build_and_isntall_current_dir from there. 283 284 Afterwards the build (regardless of failure) extracted .tar.gz 285 directory is cleaned up. 286 287 @returns True on success, False otherwise. 288 289 @raises OSError If the expected extraction directory does not exist. 290 """ 291 self._extract_compressed_package() 292 extension = self._get_extension(self.verified_package) 293 os.chdir(os.path.dirname(self.verified_package)) 294 os.chdir(self.extracted_package_path) 295 extracted_dir = os.getcwd() 296 try: 297 return self._build_and_install_current_dir(install_dir) 298 finally: 299 os.chdir(os.path.join(extracted_dir, '..')) 300 shutil.rmtree(extracted_dir) 301 302 303 def _extract_compressed_package(self): 304 """Extract the fetched compressed .tar or .zip within its directory.""" 305 if not self.verified_package: 306 raise Error('Package must have been fetched first.') 307 os.chdir(os.path.dirname(self.verified_package)) 308 if self.verified_package.endswith('gz'): 309 status = system("tar -xzf '%s'" % self.verified_package) 310 elif self.verified_package.endswith('bz2'): 311 status = system("tar -xjf '%s'" % self.verified_package) 312 elif self.verified_package.endswith('zip'): 313 status = system("unzip '%s'" % self.verified_package) 314 else: 315 raise Error('Unknown compression suffix on %s.' % 316 self.verified_package) 317 if status: 318 raise Error('tar failed with %s' % (status,)) 319 320 321 def _build_using_setup_py(self, setup_py='setup.py'): 322 """ 323 Assuming the cwd is the extracted python package, execute a simple 324 python setup.py build. 325 326 @param setup_py - The name of the setup.py file to execute. 327 328 @returns True on success, False otherwise. 329 """ 330 if not os.path.exists(setup_py): 331 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 332 status = system("'%s' %s build" % (sys.executable, setup_py)) 333 if status: 334 logging.error('%s build failed.', self.name) 335 return False 336 return True 337 338 339 def _build_egg_using_setup_py(self, setup_py='setup.py'): 340 """ 341 Assuming the cwd is the extracted python package, execute a simple 342 python setup.py bdist_egg. 343 344 @param setup_py - The name of the setup.py file to execute. 345 346 @returns The relative path to the resulting egg file or '' on failure. 347 """ 348 if not os.path.exists(setup_py): 349 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 350 egg_subdir = 'dist' 351 if os.path.isdir(egg_subdir): 352 shutil.rmtree(egg_subdir) 353 status = system("'%s' %s bdist_egg" % (sys.executable, setup_py)) 354 if status: 355 logging.error('bdist_egg of setuptools failed.') 356 return '' 357 # I've never seen a bdist_egg lay multiple .egg files. 358 for filename in os.listdir(egg_subdir): 359 if filename.endswith('.egg'): 360 return os.path.join(egg_subdir, filename) 361 362 363 def _install_from_egg(self, install_dir, egg_path): 364 """ 365 Install a module from an egg file by unzipping the necessary parts 366 into install_dir. 367 368 @param install_dir - The installation directory. 369 @param egg_path - The pathname of the egg file. 370 """ 371 status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path)) 372 if status: 373 logging.error('unzip of %s failed', egg_path) 374 return False 375 egg_info_dir = os.path.join(install_dir, 'EGG-INFO') 376 if os.path.isdir(egg_info_dir): 377 egg_info_new_path = self._get_egg_info_path(install_dir) 378 if egg_info_new_path: 379 if os.path.exists(egg_info_new_path): 380 shutil.rmtree(egg_info_new_path) 381 os.rename(egg_info_dir, egg_info_new_path) 382 else: 383 shutil.rmtree(egg_info_dir) 384 return True 385 386 387 def _get_egg_info_path(self, install_dir): 388 """Get egg-info path for this package. 389 390 Example path: install_dir/MySQL_python-1.2.3.egg-info 391 392 """ 393 if self.dist_name: 394 egg_info_name_part = self.dist_name.replace('-', '_') 395 if self.version: 396 egg_info_filename = '%s-%s.egg-info' % (egg_info_name_part, 397 self.version) 398 else: 399 egg_info_filename = '%s.egg-info' % (egg_info_name_part,) 400 return os.path.join(install_dir, egg_info_filename) 401 else: 402 return None 403 404 405 def _get_temp_dir(self): 406 return tempfile.mkdtemp(dir='/var/tmp') 407 408 409 def _site_packages_path(self, temp_dir): 410 # This makes assumptions about what python setup.py install 411 # does when given a prefix. Is this always correct? 412 python_xy = 'python%s' % sys.version[:3] 413 return os.path.join(temp_dir, 'lib', python_xy, 'site-packages') 414 415 416 def _rsync (self, temp_site_dir, install_dir): 417 """Rsync contents. """ 418 status = system("rsync -r '%s/' '%s/'" % 419 (os.path.normpath(temp_site_dir), 420 os.path.normpath(install_dir))) 421 if status: 422 logging.error('%s rsync to install_dir failed.', self.name) 423 return False 424 return True 425 426 427 def _install_using_setup_py_and_rsync(self, install_dir, 428 setup_py='setup.py', 429 temp_dir=None): 430 """ 431 Assuming the cwd is the extracted python package, execute a simple: 432 433 python setup.py install --prefix=BLA 434 435 BLA will be a temporary directory that everything installed will 436 be picked out of and rsynced to the appropriate place under 437 install_dir afterwards. 438 439 Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/ 440 directory tree that setuptools created and moves all installed 441 site-packages directly up into install_dir itself. 442 443 @param install_dir the directory for the install to happen under. 444 @param setup_py - The name of the setup.py file to execute. 445 446 @returns True on success, False otherwise. 447 """ 448 if not os.path.exists(setup_py): 449 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 450 451 if temp_dir is None: 452 temp_dir = self._get_temp_dir() 453 454 try: 455 status = system("'%s' %s install --no-compile --prefix='%s'" 456 % (sys.executable, setup_py, temp_dir)) 457 if status: 458 logging.error('%s install failed.', self.name) 459 return False 460 461 if os.path.isdir(os.path.join(temp_dir, 'lib')): 462 # NOTE: This ignores anything outside of the lib/ dir that 463 # was installed. 464 temp_site_dir = self._site_packages_path(temp_dir) 465 else: 466 temp_site_dir = temp_dir 467 468 return self._rsync(temp_site_dir, install_dir) 469 finally: 470 shutil.rmtree(temp_dir) 471 472 473 474 def _build_using_make(self, install_dir): 475 """Build the current package using configure/make. 476 477 @returns True on success, False otherwise. 478 """ 479 install_prefix = os.path.join(install_dir, 'usr', 'local') 480 status = system('./configure --prefix=%s' % install_prefix) 481 if status: 482 logging.error('./configure failed for %s', self.name) 483 return False 484 status = system('make') 485 if status: 486 logging.error('make failed for %s', self.name) 487 return False 488 status = system('make check') 489 if status: 490 logging.error('make check failed for %s', self.name) 491 return False 492 return True 493 494 495 def _install_using_make(self): 496 """Install the current package using make install. 497 498 Assumes the install path was set up while running ./configure (in 499 _build_using_make()). 500 501 @returns True on success, False otherwise. 502 """ 503 status = system('make install') 504 return status == 0 505 506 507 def fetch(self, dest_dir): 508 """ 509 Fetch the package from one its URLs and save it in dest_dir. 510 511 If the the package already exists in dest_dir and the checksum 512 matches this code will not fetch it again. 513 514 Sets the 'verified_package' attribute with the destination pathname. 515 516 @param dest_dir - The destination directory to save the local file. 517 If it does not exist it will be created. 518 519 @returns A boolean indicating if we the package is now in dest_dir. 520 @raises FetchError - When something unexpected happens. 521 """ 522 if not os.path.exists(dest_dir): 523 os.makedirs(dest_dir) 524 local_path = os.path.join(dest_dir, self.local_filename) 525 526 # If the package exists, verify its checksum and be happy if it is good. 527 if os.path.exists(local_path): 528 actual_hex_sum = _checksum_file(local_path) 529 if self.hex_sum == actual_hex_sum: 530 logging.info('Good checksum for existing %s package.', 531 self.name) 532 self.verified_package = local_path 533 return True 534 logging.warning('Bad checksum for existing %s package. ' 535 'Re-downloading', self.name) 536 os.rename(local_path, local_path + '.wrong-checksum') 537 538 # Download the package from one of its urls, rejecting any if the 539 # checksum does not match. 540 for url in self.urls: 541 logging.info('Fetching %s', url) 542 try: 543 url_file = urllib2.urlopen(url) 544 except (urllib2.URLError, EnvironmentError): 545 logging.warning('Could not fetch %s package from %s.', 546 self.name, url) 547 continue 548 549 data_length = int(url_file.info().get('Content-Length', 550 _MAX_PACKAGE_SIZE)) 551 if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE: 552 raise FetchError('%s from %s fails Content-Length %d ' 553 'sanity check.' % (self.name, url, 554 data_length)) 555 checksum = utils.hash('sha1') 556 total_read = 0 557 output = open(local_path, 'wb') 558 try: 559 while total_read < data_length: 560 data = url_file.read(_READ_SIZE) 561 if not data: 562 break 563 output.write(data) 564 checksum.update(data) 565 total_read += len(data) 566 finally: 567 output.close() 568 if self.hex_sum != checksum.hexdigest(): 569 logging.warning('Bad checksum for %s fetched from %s.', 570 self.name, url) 571 logging.warning('Got %s', checksum.hexdigest()) 572 logging.warning('Expected %s', self.hex_sum) 573 os.unlink(local_path) 574 continue 575 logging.info('Good checksum.') 576 self.verified_package = local_path 577 return True 578 else: 579 return False 580 581 582# NOTE: This class definition must come -before- all other ExternalPackage 583# classes that need to use this version of setuptools so that is is inserted 584# into the ExternalPackage.subclasses list before them. 585class SetuptoolsPackage(ExternalPackage): 586 """setuptools package""" 587 # For all known setuptools releases a string compare works for the 588 # version string. Hopefully they never release a 0.10. (Their own 589 # version comparison code would break if they did.) 590 # Any system with setuptools > 18.0.1 is fine. If none installed, then 591 # try to install the latest found on the upstream. 592 minimum_version = '18.0.1' 593 version = '18.0.1' 594 urls = ('http://pypi.python.org/packages/source/s/setuptools/' 595 'setuptools-%s.tar.gz' % (version,),) 596 local_filename = 'setuptools-%s.tar.gz' % version 597 hex_sum = 'ebc4fe81b7f6d61d923d9519f589903824044f52' 598 599 SUDO_SLEEP_DELAY = 15 600 601 602 def _build_and_install(self, install_dir): 603 """Install setuptools on the system.""" 604 logging.info('NOTE: setuptools install does not use install_dir.') 605 return self._build_and_install_from_package(install_dir) 606 607 608 def _build_and_install_current_dir(self, install_dir): 609 egg_path = self._build_egg_using_setup_py() 610 if not egg_path: 611 return False 612 613 print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n' 614 print 'About to run sudo to install setuptools', self.version 615 print 'on your system for use by', sys.executable, '\n' 616 print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n' 617 time.sleep(self.SUDO_SLEEP_DELAY) 618 619 # Copy the egg to the local filesystem /var/tmp so that root can 620 # access it properly (avoid NFS squashroot issues). 621 temp_dir = self._get_temp_dir() 622 try: 623 shutil.copy(egg_path, temp_dir) 624 egg_name = os.path.split(egg_path)[1] 625 temp_egg = os.path.join(temp_dir, egg_name) 626 p = subprocess.Popen(['sudo', '/bin/sh', temp_egg], 627 stdout=subprocess.PIPE) 628 regex = re.compile('Copying (.*?) to (.*?)\n') 629 match = regex.search(p.communicate()[0]) 630 status = p.wait() 631 632 if match: 633 compiled = os.path.join(match.group(2), match.group(1)) 634 os.system("sudo chmod a+r '%s'" % compiled) 635 finally: 636 shutil.rmtree(temp_dir) 637 638 if status: 639 logging.error('install of setuptools from egg failed.') 640 return False 641 return True 642 643 644class MySQLdbPackage(ExternalPackage): 645 """mysql package, used in scheduler.""" 646 module_name = 'MySQLdb' 647 version = '1.2.3' 648 urls = ('http://downloads.sourceforge.net/project/mysql-python/' 649 'mysql-python/%(version)s/MySQL-python-%(version)s.tar.gz' 650 % dict(version=version),) 651 local_filename = 'MySQL-python-%s.tar.gz' % version 652 hex_sum = '3511bb8c57c6016eeafa531d5c3ea4b548915e3c' 653 654 _build_and_install_current_dir = ( 655 ExternalPackage._build_and_install_current_dir_setup_py) 656 657 658 def _build_and_install(self, install_dir): 659 if not os.path.exists('/usr/bin/mysql_config'): 660 error_msg = ('You need to install /usr/bin/mysql_config.\n' 661 'On Ubuntu or Debian based systems use this: ' 662 'sudo apt-get install libmysqlclient15-dev') 663 logging.error(error_msg) 664 return False, error_msg 665 return self._build_and_install_from_package(install_dir) 666 667 668class DjangoPackage(ExternalPackage): 669 """django package.""" 670 version = '1.5.1' 671 local_filename = 'Django-%s.tar.gz' % version 672 urls = ('http://www.djangoproject.com/download/%s/tarball/' % version,) 673 hex_sum = '0ab97b90c4c79636e56337f426f1e875faccbba1' 674 675 _build_and_install = ExternalPackage._build_and_install_from_package 676 _build_and_install_current_dir = ( 677 ExternalPackage._build_and_install_current_dir_noegg) 678 679 680 def _get_installed_version_from_module(self, module): 681 try: 682 return module.get_version().split()[0] 683 except AttributeError: 684 return '0.9.6' 685 686 687 688class NumpyPackage(ExternalPackage): 689 """numpy package, required by matploglib.""" 690 version = '1.7.0' 691 local_filename = 'numpy-%s.tar.gz' % version 692 urls = ('http://downloads.sourceforge.net/project/numpy/NumPy/%(version)s/' 693 'numpy-%(version)s.tar.gz' % dict(version=version),) 694 hex_sum = 'ba328985f20390b0f969a5be2a6e1141d5752cf9' 695 696 _build_and_install = ExternalPackage._build_and_install_from_package 697 _build_and_install_current_dir = ( 698 ExternalPackage._build_and_install_current_dir_setupegg_py) 699 700 701class MatplotlibPackage(ExternalPackage): 702 """ 703 matplotlib package 704 705 This requires numpy so it must be declared after numpy to guarantee that 706 it is already installed. 707 """ 708 version = '0.98.5.3' 709 short_version = '0.98.5' 710 local_filename = 'matplotlib-%s.tar.gz' % version 711 urls = ('http://downloads.sourceforge.net/project/matplotlib/matplotlib/' 712 'matplotlib-%s/matplotlib-%s.tar.gz' % (short_version, version),) 713 hex_sum = '2f6c894cf407192b3b60351bcc6468c0385d47b6' 714 os_requirements = {('/usr/include/freetype2/ft2build.h', 715 '/usr/include/ft2build.h'): 'libfreetype6-dev', 716 ('/usr/include/png.h'): 'libpng12-dev'} 717 718 _build_and_install = ExternalPackage._build_and_install_from_package 719 _build_and_install_current_dir = ( 720 ExternalPackage._build_and_install_current_dir_setupegg_py) 721 722 723class JsonRPCLib(ExternalPackage): 724 """jsonrpclib package""" 725 version = '0.1.3' 726 module_name = 'jsonrpclib' 727 local_filename = '%s-%s.tar.gz' % (module_name, version) 728 urls = ('http://pypi.python.org/packages/source/j/%s/%s' % 729 (module_name, local_filename), ) 730 hex_sum = '431714ed19ab677f641ce5d678a6a95016f5c452' 731 732 def _get_installed_version_from_module(self, module): 733 # jsonrpclib doesn't contain a proper version 734 return self.version 735 736 _build_and_install = ExternalPackage._build_and_install_from_package 737 _build_and_install_current_dir = ( 738 ExternalPackage._build_and_install_current_dir_noegg) 739 740 741class Httplib2Package(ExternalPackage): 742 """httplib2 package""" 743 version = '0.6.0' 744 local_filename = 'httplib2-%s.tar.gz' % version 745 # Cannot use the newest httplib2 package 0.9.2 since it cannot be installed 746 # directly in a temp folder. So keep it as 0.6.0. 747 urls = ('https://launchpad.net/ubuntu/+archive/primary/+files/' 748 'python-httplib2_' + version + '.orig.tar.gz',) 749 hex_sum = '995344b2704826cc0d61a266e995b328d92445a5' 750 751 def _get_installed_version_from_module(self, module): 752 # httplib2 doesn't contain a proper version 753 return self.version 754 755 _build_and_install = ExternalPackage._build_and_install_from_package 756 _build_and_install_current_dir = ( 757 ExternalPackage._build_and_install_current_dir_noegg) 758 759 760class GwtPackage(ExternalPackage): 761 """Fetch and extract a local copy of GWT used to build the frontend.""" 762 763 version = '2.3.0' 764 local_filename = 'gwt-%s.zip' % version 765 urls = ('https://storage.googleapis.com/google-code-archive-downloads/' 766 'v2/code.google.com/google-web-toolkit/' + local_filename,) 767 hex_sum = 'd51fce9166e6b31349659ffca89baf93e39bc84b' 768 name = 'gwt' 769 about_filename = 'about.txt' 770 module_name = None # Not a Python module. 771 772 773 def is_needed(self, install_dir): 774 gwt_dir = os.path.join(install_dir, self.name) 775 about_file = os.path.join(install_dir, self.name, self.about_filename) 776 777 if not os.path.exists(gwt_dir) or not os.path.exists(about_file): 778 logging.info('gwt not installed for autotest') 779 return True 780 781 f = open(about_file, 'r') 782 version_line = f.readline() 783 f.close() 784 785 match = re.match(r'Google Web Toolkit (.*)', version_line) 786 if not match: 787 logging.info('did not find gwt version') 788 return True 789 790 logging.info('found gwt version %s', match.group(1)) 791 return match.group(1) != self.version 792 793 794 def _build_and_install(self, install_dir): 795 os.chdir(install_dir) 796 self._extract_compressed_package() 797 extracted_dir = self.local_filename[:-len('.zip')] 798 target_dir = os.path.join(install_dir, self.name) 799 if os.path.exists(target_dir): 800 shutil.rmtree(target_dir) 801 os.rename(extracted_dir, target_dir) 802 return True 803 804 805class GVizAPIPackage(ExternalPackage): 806 """gviz package""" 807 module_name = 'gviz_api' 808 version = '1.8.2' 809 local_filename = 'google-visualization-python.zip' 810 urls = ('https://github.com/google/google-visualization-python/' 811 'archive/master.zip',) 812 hex_sum = 'ec70fb8b874eae21e331332065415318f6fe4882' 813 extracted_package_path = 'google-visualization-python-master' 814 815 _build_and_install = ExternalPackage._build_and_install_from_package 816 _build_and_install_current_dir = ( 817 ExternalPackage._build_and_install_current_dir_noegg) 818 819 def _get_installed_version_from_module(self, module): 820 # gviz doesn't contain a proper version 821 return self.version 822 823 824class StatsdPackage(ExternalPackage): 825 """python-statsd package""" 826 version = '1.7.2' 827 url_filename = 'python-statsd-%s.tar.gz' % version 828 local_filename = url_filename 829 urls = ('http://pypi.python.org/packages/source/p/python-statsd/%s' % ( 830 url_filename),) 831 hex_sum = '2cc186ebdb723e2420b432ab71639786d877694b' 832 833 _build_and_install = ExternalPackage._build_and_install_from_package 834 _build_and_install_current_dir = ( 835 ExternalPackage._build_and_install_current_dir_setup_py) 836 837 838class GdataPackage(ExternalPackage): 839 """ 840 Pulls the GData library, giving us an API to query tracker. 841 """ 842 version = '2.0.18' 843 local_filename = 'gdata-%s.zip' % version 844 urls = ('https://github.com/google/gdata-python-client/' + 845 'archive/master.zip',) 846 hex_sum = '893f9c9f627ef92afe8f3f066311d9b3748f1732' 847 extracted_package_path = 'gdata-python-client-master' 848 849 _build_and_install = ExternalPackage._build_and_install_from_package 850 _build_and_install_current_dir = ( 851 ExternalPackage._build_and_install_current_dir_noegg) 852 853 def _get_installed_version_from_module(self, module): 854 # gdata doesn't contain a proper version 855 return self.version 856 857 858class DnsPythonPackage(ExternalPackage): 859 """ 860 dns module 861 862 Used in unittests. 863 """ 864 module_name = 'dns' 865 version = '1.3.5' 866 url_filename = 'dnspython-%s.tar.gz' % version 867 local_filename = url_filename 868 urls = ('http://www.dnspython.org/kits/%s/%s' % ( 869 version, url_filename),) 870 871 hex_sum = '06314dad339549613435470c6add992910e26e5d' 872 873 _build_and_install = ExternalPackage._build_and_install_from_package 874 _build_and_install_current_dir = ( 875 ExternalPackage._build_and_install_current_dir_noegg) 876 877 def _get_installed_version_from_module(self, module): 878 """Ask our module its version string and return it or '' if unknown.""" 879 try: 880 __import__(self.module_name + '.version') 881 return module.version.version 882 except AttributeError: 883 logging.error('could not get version from %s', module) 884 return '' 885 886 887class PyudevPackage(ExternalPackage): 888 """ 889 pyudev module 890 891 Used in unittests. 892 """ 893 version = '0.16.1' 894 url_filename = 'pyudev-%s.tar.gz' % version 895 local_filename = url_filename 896 urls = ('http://pypi.python.org/packages/source/p/pyudev/%s' % ( 897 url_filename),) 898 hex_sum = 'b36bc5c553ce9b56d32a5e45063a2c88156771c0' 899 900 _build_and_install = ExternalPackage._build_and_install_from_package 901 _build_and_install_current_dir = ( 902 ExternalPackage._build_and_install_current_dir_setup_py) 903 904 905class PyMoxPackage(ExternalPackage): 906 """ 907 mox module 908 909 Used in unittests. 910 """ 911 module_name = 'mox' 912 version = '0.5.3' 913 url_filename = 'mox-%s.tar.gz' % version 914 local_filename = url_filename 915 urls = ('http://pypi.python.org/packages/source/m/mox/%s' % ( 916 url_filename),) 917 hex_sum = '1c502d2c0a8aefbba2c7f385a83d33e7d822452a' 918 919 _build_and_install = ExternalPackage._build_and_install_from_package 920 _build_and_install_current_dir = ( 921 ExternalPackage._build_and_install_current_dir_noegg) 922 923 def _get_installed_version_from_module(self, module): 924 # mox doesn't contain a proper version 925 return self.version 926 927 928class MockPackage(ExternalPackage): 929 """ 930 mock module 931 932 Used in unittests. 933 """ 934 module_name = 'mock' 935 version = '2.0.0' 936 url_filename = 'mock-%s.tar.gz' % version 937 local_filename = url_filename 938 urls = ('http://pypi.python.org/packages/source/m/mock/%s' % ( 939 url_filename),) 940 hex_sum = '397ed52eb2d8d4b326bc3fa6b38adda5f0b090d3' 941 942 _build_and_install = ExternalPackage._build_and_install_from_package 943 _build_and_install_current_dir = ( 944 ExternalPackage._build_and_install_current_dir_noegg) 945 946 947class PySeleniumPackage(ExternalPackage): 948 """ 949 selenium module 950 951 Used in wifi_interop suite. 952 """ 953 module_name = 'selenium' 954 version = '2.37.2' 955 url_filename = 'selenium-%s.tar.gz' % version 956 local_filename = url_filename 957 urls = ('https://pypi.python.org/packages/source/s/selenium/%s' % ( 958 url_filename),) 959 hex_sum = '66946d5349e36d946daaad625c83c30c11609e36' 960 961 _build_and_install = ExternalPackage._build_and_install_from_package 962 _build_and_install_current_dir = ( 963 ExternalPackage._build_and_install_current_dir_setup_py) 964 965 966class FaultHandlerPackage(ExternalPackage): 967 """ 968 faulthandler module 969 """ 970 module_name = 'faulthandler' 971 version = '2.3' 972 url_filename = '%s-%s.tar.gz' % (module_name, version) 973 local_filename = url_filename 974 urls = ('http://pypi.python.org/packages/source/f/faulthandler/%s' % 975 (url_filename),) 976 hex_sum = 'efb30c068414fba9df892e48fcf86170cbf53589' 977 978 _build_and_install = ExternalPackage._build_and_install_from_package 979 _build_and_install_current_dir = ( 980 ExternalPackage._build_and_install_current_dir_noegg) 981 982 983class PsutilPackage(ExternalPackage): 984 """ 985 psutil module 986 """ 987 module_name = 'psutil' 988 version = '2.1.1' 989 url_filename = '%s-%s.tar.gz' % (module_name, version) 990 local_filename = url_filename 991 urls = ('http://pypi.python.org/packages/source/p/psutil/%s' % 992 (url_filename),) 993 hex_sum = '0c20a20ed316e69f2b0881530439213988229916' 994 995 _build_and_install = ExternalPackage._build_and_install_from_package 996 _build_and_install_current_dir = ( 997 ExternalPackage._build_and_install_current_dir_setup_py) 998 999 1000class ElasticSearchPackage(ExternalPackage): 1001 """elasticsearch-py package.""" 1002 version = '1.6.0' 1003 url_filename = 'elasticsearch-%s.tar.gz' % version 1004 local_filename = url_filename 1005 urls = ('https://pypi.python.org/packages/source/e/elasticsearch/%s' % 1006 (url_filename),) 1007 hex_sum = '3e676c96f47935b1f52df82df3969564bd356b1c' 1008 _build_and_install = ExternalPackage._build_and_install_from_package 1009 _build_and_install_current_dir = ( 1010 ExternalPackage._build_and_install_current_dir_setup_py) 1011 1012 def _get_installed_version_from_module(self, module): 1013 # Elastic's version format is like tuple (1, 6, 0), which needs to be 1014 # transferred to 1.6.0. 1015 try: 1016 return '.'.join(str(i) for i in module.__version__) 1017 except: 1018 return self.version 1019 1020 1021class Urllib3Package(ExternalPackage): 1022 """elasticsearch-py package.""" 1023 version = '1.9' 1024 url_filename = 'urllib3-%s.tar.gz' % version 1025 local_filename = url_filename 1026 urls = ('https://pypi.python.org/packages/source/u/urllib3/%s' % 1027 (url_filename),) 1028 hex_sum = '9522197efb2a2b49ce804de3a515f06d97b6602f' 1029 _build_and_install = ExternalPackage._build_and_install_from_package 1030 _build_and_install_current_dir = ( 1031 ExternalPackage._build_and_install_current_dir_setup_py) 1032 1033 1034class ImagingLibraryPackage(ExternalPackage): 1035 """Python Imaging Library (PIL).""" 1036 version = '1.1.7' 1037 url_filename = 'Imaging-%s.tar.gz' % version 1038 local_filename = url_filename 1039 urls = ('http://effbot.org/downloads/%s' % url_filename,) 1040 hex_sum = '76c37504251171fda8da8e63ecb8bc42a69a5c81' 1041 1042 def _build_and_install(self, install_dir): 1043 # The path of zlib library might be different from what PIL setup.py is 1044 # expected. Following change does the best attempt to link the library 1045 # to a path PIL setup.py will try. 1046 libz_possible_path = '/usr/lib/x86_64-linux-gnu/libz.so' 1047 libz_expected_path = '/usr/lib/libz.so' 1048 if (os.path.exists(libz_possible_path) and 1049 not os.path.exists(libz_expected_path)): 1050 utils.run('sudo ln -s %s %s' % 1051 (libz_possible_path, libz_expected_path)) 1052 return self._build_and_install_from_package(install_dir) 1053 1054 _build_and_install_current_dir = ( 1055 ExternalPackage._build_and_install_current_dir_noegg) 1056 1057 1058class AstroidPackage(ExternalPackage): 1059 """astroid package.""" 1060 version = '1.0.0' 1061 url_filename = 'astroid-%s.tar.gz' % version 1062 local_filename = url_filename 1063 #md5=e74430dfbbe09cd18ef75bd76f95425a 1064 urls = ('https://pypi.python.org/packages/15/ef/' 1065 '1c01161c40ce08451254125935c5bca85b08913e610a4708760ee1432fa8/%s' % 1066 (url_filename),) 1067 hex_sum = '2ebba76d115cb8a2d84d8777d8535ddac86daaa6' 1068 _build_and_install = ExternalPackage._build_and_install_from_package 1069 _build_and_install_current_dir = ( 1070 ExternalPackage._build_and_install_current_dir_setup_py) 1071 1072 1073class LogilabCommonPackage(ExternalPackage): 1074 """logilab-common package.""" 1075 version = '1.2.2' 1076 module_name = 'logilab' 1077 url_filename = 'logilab-common-%s.tar.gz' % version 1078 local_filename = url_filename 1079 #md5=daa7b20c8374ff5f525882cf67e258c0 1080 urls = ('https://pypi.python.org/packages/63/5b/' 1081 'd4d93ad9e683a06354bc5893194514fbf5d05ef86b06b0285762c3724509/%s' % 1082 (url_filename),) 1083 hex_sum = 'ecad2d10c31dcf183c8bed87b6ec35e7ed397d27' 1084 _build_and_install = ExternalPackage._build_and_install_from_package 1085 _build_and_install_current_dir = ( 1086 ExternalPackage._build_and_install_current_dir_setup_py) 1087 1088 1089class PyLintPackage(ExternalPackage): 1090 """pylint package.""" 1091 version = '1.1.0' 1092 url_filename = 'pylint-%s.tar.gz' % version 1093 local_filename = url_filename 1094 #md5=017299b5911838a9347a71de5f946afc 1095 urls = ('https://pypi.python.org/packages/09/69/' 1096 'cf252f211dbbf58bbbe01a3931092d8a8df8d55f5fe23ac5cef145aa6468/%s' % 1097 (url_filename),) 1098 hex_sum = 'b33594a2c627d72007bfa8c6d7619af699e26085' 1099 _build_and_install = ExternalPackage._build_and_install_from_package 1100 _build_and_install_current_dir = ( 1101 ExternalPackage._build_and_install_current_dir_setup_py) 1102 1103 1104class Pytz(ExternalPackage): 1105 """Pytz package.""" 1106 version = '2016.10' 1107 url_filename = 'pytz-%s.tar.gz' % version 1108 local_filename = url_filename 1109 #md5=cc9f16ba436efabdcef3c4d32ae4919c 1110 urls = ('https://pypi.python.org/packages/42/00/' 1111 '5c89fc6c9b305df84def61863528e899e9dccb196f8438f6cbe960758fc5/%s' % 1112 (url_filename),) 1113 hex_sum = '8d63f1e9b1ee862841b990a7d8ad1d4508d9f0be' 1114 _build_and_install = ExternalPackage._build_and_install_from_package 1115 _build_and_install_current_dir = ( 1116 ExternalPackage._build_and_install_current_dir_setup_py) 1117 1118 1119class Tzlocal(ExternalPackage): 1120 """Tzlocal package.""" 1121 version = '1.3' 1122 url_filename = 'tzlocal-%s.tar.gz' % version 1123 local_filename = url_filename 1124 # md5=3cb544b3975b59f91a793850a072d4a8 1125 urls = ('https://pypi.python.org/packages/d3/64/' 1126 'e4b18738496213f82b88b31c431a0e4ece143801fb6771dddd1c2bf0101b/%s' % 1127 (url_filename),) 1128 hex_sum = '730e9d7112335865a1dcfabec69c8c3086be424f' 1129 _build_and_install = ExternalPackage._build_and_install_from_package 1130 _build_and_install_current_dir = ( 1131 ExternalPackage._build_and_install_current_dir_setup_py) 1132 1133 1134class _ExternalGitRepo(ExternalPackage): 1135 """ 1136 Parent class for any package which needs to pull a git repo. 1137 1138 This class inherits from ExternalPackage only so we can sync git 1139 repos through the build_externals script. We do not reuse any of 1140 ExternalPackage's other methods. Any package that needs a git repo 1141 should subclass this and override build_and_install or fetch as 1142 they see appropriate. 1143 """ 1144 1145 os_requirements = {('/usr/bin/git') : 'git-core'} 1146 1147 # All the chromiumos projects used on the lab servers should have a 'prod' 1148 # branch used to track the software version deployed in prod. 1149 PROD_BRANCH = 'prod' 1150 MASTER_BRANCH = 'master' 1151 1152 def is_needed(self, unused_install_dir): 1153 """Tell build_externals that we need to fetch.""" 1154 # TODO(beeps): check if we're already upto date. 1155 return True 1156 1157 1158 def build_and_install(self, unused_install_dir): 1159 """ 1160 Fall through method to install a package. 1161 1162 Overwritten in base classes to pull a git repo. 1163 """ 1164 raise NotImplementedError 1165 1166 1167 def fetch(self, unused_dest_dir): 1168 """Fallthrough method to fetch a package.""" 1169 return True 1170 1171 1172class HdctoolsRepo(_ExternalGitRepo): 1173 """Clones or updates the hdctools repo.""" 1174 1175 module_name = 'servo' 1176 temp_hdctools_dir = tempfile.mktemp(suffix='hdctools') 1177 _GIT_URL = ('https://chromium.googlesource.com/' 1178 'chromiumos/third_party/hdctools') 1179 1180 def fetch(self, unused_dest_dir): 1181 """ 1182 Fetch repo to a temporary location. 1183 1184 We use an intermediate temp directory to stage our 1185 installation because we only care about the servo package. 1186 If we can't get at the top commit hash after fetching 1187 something is wrong. This can happen when we've cloned/pulled 1188 an empty repo. Not something we expect to do. 1189 1190 @parma unused_dest_dir: passed in because we inherit from 1191 ExternalPackage. 1192 1193 @return: True if repo sync was successful. 1194 """ 1195 git_repo = revision_control.GitRepo( 1196 self.temp_hdctools_dir, 1197 self._GIT_URL, 1198 None, 1199 abs_work_tree=self.temp_hdctools_dir) 1200 git_repo.reinit_repo_at(self.PROD_BRANCH) 1201 1202 if git_repo.get_latest_commit_hash(): 1203 return True 1204 return False 1205 1206 1207 def build_and_install(self, install_dir): 1208 """Reach into the hdctools repo and rsync only the servo directory.""" 1209 1210 servo_dir = os.path.join(self.temp_hdctools_dir, 'servo') 1211 if not os.path.exists(servo_dir): 1212 return False 1213 1214 rv = self._rsync(servo_dir, os.path.join(install_dir, 'servo')) 1215 shutil.rmtree(self.temp_hdctools_dir) 1216 return rv 1217 1218 1219class ChromiteRepo(_ExternalGitRepo): 1220 """Clones or updates the chromite repo.""" 1221 1222 _GIT_URL = ('https://chromium.googlesource.com/chromiumos/chromite') 1223 1224 def build_and_install(self, install_dir, master_branch=False): 1225 """ 1226 Clone if the repo isn't initialized, pull clean bits if it is. 1227 1228 Unlike it's hdctools counterpart the chromite repo clones master 1229 directly into site-packages. It doesn't use an intermediate temp 1230 directory because it doesn't need installation. 1231 1232 @param install_dir: destination directory for chromite installation. 1233 @param master_branch: if True, install master branch. Otherwise, 1234 install prod branch. 1235 """ 1236 init_branch = (self.MASTER_BRANCH if master_branch 1237 else self.PROD_BRANCH) 1238 local_chromite_dir = os.path.join(install_dir, 'chromite') 1239 git_repo = revision_control.GitRepo( 1240 local_chromite_dir, 1241 self._GIT_URL, 1242 abs_work_tree=local_chromite_dir) 1243 git_repo.reinit_repo_at(init_branch) 1244 1245 1246 if git_repo.get_latest_commit_hash(): 1247 return True 1248 return False 1249 1250 1251class DevServerRepo(_ExternalGitRepo): 1252 """Clones or updates the chromite repo.""" 1253 1254 _GIT_URL = ('https://chromium.googlesource.com/' 1255 'chromiumos/platform/dev-util') 1256 1257 def build_and_install(self, install_dir): 1258 """ 1259 Clone if the repo isn't initialized, pull clean bits if it is. 1260 1261 Unlike it's hdctools counterpart the dev-util repo clones master 1262 directly into site-packages. It doesn't use an intermediate temp 1263 directory because it doesn't need installation. 1264 1265 @param install_dir: destination directory for chromite installation. 1266 """ 1267 local_devserver_dir = os.path.join(install_dir, 'devserver') 1268 git_repo = revision_control.GitRepo(local_devserver_dir, self._GIT_URL, 1269 abs_work_tree=local_devserver_dir) 1270 git_repo.reinit_repo_at(self.PROD_BRANCH) 1271 1272 if git_repo.get_latest_commit_hash(): 1273 return True 1274 return False 1275 1276 1277class BtsocketRepo(_ExternalGitRepo): 1278 """Clones or updates the btsocket repo.""" 1279 1280 _GIT_URL = ('https://chromium.googlesource.com/' 1281 'chromiumos/platform/btsocket') 1282 1283 def fetch(self, unused_dest_dir): 1284 """ 1285 Fetch repo to a temporary location. 1286 1287 We use an intermediate temp directory because we have to build an 1288 egg for installation. If we can't get at the top commit hash after 1289 fetching something is wrong. This can happen when we've cloned/pulled 1290 an empty repo. Not something we expect to do. 1291 1292 @parma unused_dest_dir: passed in because we inherit from 1293 ExternalPackage. 1294 1295 @return: True if repo sync was successful. 1296 """ 1297 self.temp_btsocket_dir = autotemp.tempdir(unique_id='btsocket') 1298 try: 1299 git_repo = revision_control.GitRepo( 1300 self.temp_btsocket_dir.name, 1301 self._GIT_URL, 1302 None, 1303 abs_work_tree=self.temp_btsocket_dir.name) 1304 git_repo.reinit_repo_at(self.PROD_BRANCH) 1305 1306 if git_repo.get_latest_commit_hash(): 1307 return True 1308 except: 1309 self.temp_btsocket_dir.clean() 1310 raise 1311 1312 self.temp_btsocket_dir.clean() 1313 return False 1314 1315 1316 def build_and_install(self, install_dir): 1317 """ 1318 Install the btsocket module using setup.py 1319 1320 @param install_dir: Target installation directory. 1321 1322 @return: A boolean indicating success of failure. 1323 """ 1324 work_dir = os.getcwd() 1325 try: 1326 os.chdir(self.temp_btsocket_dir.name) 1327 rv = self._build_and_install_current_dir_setup_py(install_dir) 1328 finally: 1329 os.chdir(work_dir) 1330 self.temp_btsocket_dir.clean() 1331 return rv 1332