1# Please keep this code python 2.4 compatible and stand alone. 2 3import logging, os, shutil, sys, tempfile, time, urllib2 4import subprocess, re 5from autotest_lib.client.common_lib import autotemp, revision_control, utils 6 7_READ_SIZE = 64*1024 8_MAX_PACKAGE_SIZE = 100*1024*1024 9 10 11class Error(Exception): 12 """Local exception to be raised by code in this file.""" 13 14class FetchError(Error): 15 """Failed to fetch a package from any of its listed URLs.""" 16 17 18def _checksum_file(full_path): 19 """@returns The hex checksum of a file given its pathname.""" 20 inputfile = open(full_path, 'rb') 21 try: 22 hex_sum = utils.hash('sha1', inputfile.read()).hexdigest() 23 finally: 24 inputfile.close() 25 return hex_sum 26 27 28def system(commandline): 29 """Same as os.system(commandline) but logs the command first. 30 31 @param commandline: commandline to be called. 32 """ 33 logging.info(commandline) 34 return os.system(commandline) 35 36 37def find_top_of_autotest_tree(): 38 """@returns The full path to the top of the autotest directory tree.""" 39 dirname = os.path.dirname(__file__) 40 autotest_dir = os.path.abspath(os.path.join(dirname, '..')) 41 return autotest_dir 42 43 44class ExternalPackage(object): 45 """ 46 Defines an external package with URLs to fetch its sources from and 47 a build_and_install() method to unpack it, build it and install it 48 beneath our own autotest/site-packages directory. 49 50 Base Class. Subclass this to define packages. 51 Note: Unless your subclass has a specific reason to, it should not 52 re-install the package every time build_externals is invoked, as this 53 happens periodically through the scheduler. To avoid doing so the is_needed 54 method needs to return an appropriate value. 55 56 Attributes: 57 @attribute urls - A tuple of URLs to try fetching the package from. 58 @attribute local_filename - A local filename to use when saving the 59 fetched package. 60 @attribute hex_sum - The hex digest (currently SHA1) of this package 61 to be used to verify its contents. 62 @attribute module_name - The installed python module name to be used for 63 for a version check. Defaults to the lower case class name with 64 the word Package stripped off. 65 @attribute version - The desired minimum package version. 66 @attribute os_requirements - A dictionary mapping pathname tuples on the 67 the OS distribution to a likely name of a package the user 68 needs to install on their system in order to get this file. 69 One of the files in the tuple must exist. 70 @attribute name - Read only, the printable name of the package. 71 @attribute subclasses - This class attribute holds a list of all defined 72 subclasses. It is constructed dynamically using the metaclass. 73 """ 74 # Modules that are meant to be installed in system directory, rather than 75 # autotest/site-packages. These modules should be skipped if the module 76 # is already installed in system directory. This prevents an older version 77 # of the module from being installed in system directory. 78 SYSTEM_MODULES = ['setuptools'] 79 80 subclasses = [] 81 urls = () 82 local_filename = None 83 hex_sum = None 84 module_name = None 85 version = None 86 os_requirements = None 87 88 89 class __metaclass__(type): 90 """Any time a subclass is defined, add it to our list.""" 91 def __init__(mcs, name, bases, dict): 92 if name != 'ExternalPackage' and not name.startswith('_'): 93 mcs.subclasses.append(mcs) 94 95 96 def __init__(self): 97 self.verified_package = '' 98 if not self.module_name: 99 self.module_name = self.name.lower() 100 self.installed_version = '' 101 102 103 @property 104 def name(self): 105 """Return the class name with any trailing 'Package' stripped off.""" 106 class_name = self.__class__.__name__ 107 if class_name.endswith('Package'): 108 return class_name[:-len('Package')] 109 return class_name 110 111 112 def is_needed(self, install_dir): 113 """ 114 Check to see if we need to reinstall a package. This is contingent on: 115 1. Module name: If the name of the module is different from the package, 116 the class that installs it needs to specify a module_name string, 117 so we can try importing the module. 118 119 2. Installed version: If the module doesn't contain a __version__ the 120 class that installs it needs to override the 121 _get_installed_version_from_module method to return an appropriate 122 version string. 123 124 3. Version/Minimum version: The class that installs the package should 125 contain a version string, and an optional minimum version string. 126 127 4. install_dir: If the module exists in a different directory, e.g., 128 /usr/lib/python2.7/dist-packages/, the module will be forced to be 129 installed in install_dir. 130 131 @param install_dir: install directory. 132 @returns True if self.module_name needs to be built and installed. 133 """ 134 if not self.module_name or not self.version: 135 logging.warning('version and module_name required for ' 136 'is_needed() check to work.') 137 return True 138 try: 139 module = __import__(self.module_name) 140 except ImportError, e: 141 logging.info("%s isn't present. Will install.", self.module_name) 142 return True 143 if (not module.__file__.startswith(install_dir) and 144 not self.module_name in self.SYSTEM_MODULES): 145 logging.info('Module %s is installed in %s, rather than %s. The ' 146 'module will be forced to be installed in %s.', 147 self.module_name, module.__file__, install_dir, 148 install_dir) 149 return True 150 self.installed_version = self._get_installed_version_from_module(module) 151 logging.info('imported %s version %s.', self.module_name, 152 self.installed_version) 153 if hasattr(self, 'minimum_version'): 154 return self.minimum_version > self.installed_version 155 else: 156 return self.version > self.installed_version 157 158 159 def _get_installed_version_from_module(self, module): 160 """Ask our module its version string and return it or '' if unknown.""" 161 try: 162 return module.__version__ 163 except AttributeError: 164 logging.error('could not get version from %s', module) 165 return '' 166 167 168 def _build_and_install(self, install_dir): 169 """Subclasses MUST provide their own implementation.""" 170 raise NotImplementedError 171 172 173 def _build_and_install_current_dir(self, install_dir): 174 """ 175 Subclasses that use _build_and_install_from_package() MUST provide 176 their own implementation of this method. 177 """ 178 raise NotImplementedError 179 180 181 def build_and_install(self, install_dir): 182 """ 183 Builds and installs the package. It must have been fetched already. 184 185 @param install_dir - The package installation directory. If it does 186 not exist it will be created. 187 """ 188 if not self.verified_package: 189 raise Error('Must call fetch() first. - %s' % self.name) 190 self._check_os_requirements() 191 return self._build_and_install(install_dir) 192 193 194 def _check_os_requirements(self): 195 if not self.os_requirements: 196 return 197 failed = False 198 for file_names, package_name in self.os_requirements.iteritems(): 199 if not any(os.path.exists(file_name) for file_name in file_names): 200 failed = True 201 logging.error('Can\'t find %s, %s probably needs it.', 202 ' or '.join(file_names), self.name) 203 logging.error('Perhaps you need to install something similar ' 204 'to the %s package for OS first.', package_name) 205 if failed: 206 raise Error('Missing OS requirements for %s. (see above)' % 207 self.name) 208 209 210 def _build_and_install_current_dir_setup_py(self, install_dir): 211 """For use as a _build_and_install_current_dir implementation.""" 212 egg_path = self._build_egg_using_setup_py(setup_py='setup.py') 213 if not egg_path: 214 return False 215 return self._install_from_egg(install_dir, egg_path) 216 217 218 def _build_and_install_current_dir_setupegg_py(self, install_dir): 219 """For use as a _build_and_install_current_dir implementation.""" 220 egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py') 221 if not egg_path: 222 return False 223 return self._install_from_egg(install_dir, egg_path) 224 225 226 def _build_and_install_current_dir_noegg(self, install_dir): 227 if not self._build_using_setup_py(): 228 return False 229 return self._install_using_setup_py_and_rsync(install_dir) 230 231 232 def _build_and_install_from_package(self, install_dir): 233 """ 234 This method may be used as a _build_and_install() implementation 235 for subclasses if they implement _build_and_install_current_dir(). 236 237 Extracts the .tar.gz file, chdirs into the extracted directory 238 (which is assumed to match the tar filename) and calls 239 _build_and_isntall_current_dir from there. 240 241 Afterwards the build (regardless of failure) extracted .tar.gz 242 directory is cleaned up. 243 244 @returns True on success, False otherwise. 245 246 @raises OSError If the expected extraction directory does not exist. 247 """ 248 self._extract_compressed_package() 249 if self.verified_package.endswith('.tar.gz'): 250 extension = '.tar.gz' 251 elif self.verified_package.endswith('.tar.bz2'): 252 extension = '.tar.bz2' 253 elif self.verified_package.endswith('.zip'): 254 extension = '.zip' 255 else: 256 raise Error('Unexpected package file extension on %s' % 257 self.verified_package) 258 os.chdir(os.path.dirname(self.verified_package)) 259 os.chdir(self.local_filename[:-len(extension)]) 260 extracted_dir = os.getcwd() 261 try: 262 return self._build_and_install_current_dir(install_dir) 263 finally: 264 os.chdir(os.path.join(extracted_dir, '..')) 265 shutil.rmtree(extracted_dir) 266 267 268 def _extract_compressed_package(self): 269 """Extract the fetched compressed .tar or .zip within its directory.""" 270 if not self.verified_package: 271 raise Error('Package must have been fetched first.') 272 os.chdir(os.path.dirname(self.verified_package)) 273 if self.verified_package.endswith('gz'): 274 status = system("tar -xzf '%s'" % self.verified_package) 275 elif self.verified_package.endswith('bz2'): 276 status = system("tar -xjf '%s'" % self.verified_package) 277 elif self.verified_package.endswith('zip'): 278 status = system("unzip '%s'" % self.verified_package) 279 else: 280 raise Error('Unknown compression suffix on %s.' % 281 self.verified_package) 282 if status: 283 raise Error('tar failed with %s' % (status,)) 284 285 286 def _build_using_setup_py(self, setup_py='setup.py'): 287 """ 288 Assuming the cwd is the extracted python package, execute a simple 289 python setup.py build. 290 291 @param setup_py - The name of the setup.py file to execute. 292 293 @returns True on success, False otherwise. 294 """ 295 if not os.path.exists(setup_py): 296 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 297 status = system("'%s' %s build" % (sys.executable, setup_py)) 298 if status: 299 logging.error('%s build failed.', self.name) 300 return False 301 return True 302 303 304 def _build_egg_using_setup_py(self, setup_py='setup.py'): 305 """ 306 Assuming the cwd is the extracted python package, execute a simple 307 python setup.py bdist_egg. 308 309 @param setup_py - The name of the setup.py file to execute. 310 311 @returns The relative path to the resulting egg file or '' on failure. 312 """ 313 if not os.path.exists(setup_py): 314 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 315 egg_subdir = 'dist' 316 if os.path.isdir(egg_subdir): 317 shutil.rmtree(egg_subdir) 318 status = system("'%s' %s bdist_egg" % (sys.executable, setup_py)) 319 if status: 320 logging.error('bdist_egg of setuptools failed.') 321 return '' 322 # I've never seen a bdist_egg lay multiple .egg files. 323 for filename in os.listdir(egg_subdir): 324 if filename.endswith('.egg'): 325 return os.path.join(egg_subdir, filename) 326 327 328 def _install_from_egg(self, install_dir, egg_path): 329 """ 330 Install a module from an egg file by unzipping the necessary parts 331 into install_dir. 332 333 @param install_dir - The installation directory. 334 @param egg_path - The pathname of the egg file. 335 """ 336 status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path)) 337 if status: 338 logging.error('unzip of %s failed', egg_path) 339 return False 340 egg_info = os.path.join(install_dir, 'EGG-INFO') 341 if os.path.isdir(egg_info): 342 shutil.rmtree(egg_info) 343 return True 344 345 346 def _get_temp_dir(self): 347 return tempfile.mkdtemp(dir='/var/tmp') 348 349 350 def _site_packages_path(self, temp_dir): 351 # This makes assumptions about what python setup.py install 352 # does when given a prefix. Is this always correct? 353 python_xy = 'python%s' % sys.version[:3] 354 return os.path.join(temp_dir, 'lib', python_xy, 'site-packages') 355 356 357 def _rsync (self, temp_site_dir, install_dir): 358 """Rsync contents. """ 359 status = system("rsync -r '%s/' '%s/'" % 360 (os.path.normpath(temp_site_dir), 361 os.path.normpath(install_dir))) 362 if status: 363 logging.error('%s rsync to install_dir failed.', self.name) 364 return False 365 return True 366 367 368 def _install_using_setup_py_and_rsync(self, install_dir, 369 setup_py='setup.py', 370 temp_dir=None): 371 """ 372 Assuming the cwd is the extracted python package, execute a simple: 373 374 python setup.py install --prefix=BLA 375 376 BLA will be a temporary directory that everything installed will 377 be picked out of and rsynced to the appropriate place under 378 install_dir afterwards. 379 380 Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/ 381 directory tree that setuptools created and moves all installed 382 site-packages directly up into install_dir itself. 383 384 @param install_dir the directory for the install to happen under. 385 @param setup_py - The name of the setup.py file to execute. 386 387 @returns True on success, False otherwise. 388 """ 389 if not os.path.exists(setup_py): 390 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 391 392 if temp_dir is None: 393 temp_dir = self._get_temp_dir() 394 395 try: 396 status = system("'%s' %s install --no-compile --prefix='%s'" 397 % (sys.executable, setup_py, temp_dir)) 398 if status: 399 logging.error('%s install failed.', self.name) 400 return False 401 402 if os.path.isdir(os.path.join(temp_dir, 'lib')): 403 # NOTE: This ignores anything outside of the lib/ dir that 404 # was installed. 405 temp_site_dir = self._site_packages_path(temp_dir) 406 else: 407 temp_site_dir = temp_dir 408 409 return self._rsync(temp_site_dir, install_dir) 410 finally: 411 shutil.rmtree(temp_dir) 412 413 414 415 def _build_using_make(self, install_dir): 416 """Build the current package using configure/make. 417 418 @returns True on success, False otherwise. 419 """ 420 install_prefix = os.path.join(install_dir, 'usr', 'local') 421 status = system('./configure --prefix=%s' % install_prefix) 422 if status: 423 logging.error('./configure failed for %s', self.name) 424 return False 425 status = system('make') 426 if status: 427 logging.error('make failed for %s', self.name) 428 return False 429 status = system('make check') 430 if status: 431 logging.error('make check failed for %s', self.name) 432 return False 433 return True 434 435 436 def _install_using_make(self): 437 """Install the current package using make install. 438 439 Assumes the install path was set up while running ./configure (in 440 _build_using_make()). 441 442 @returns True on success, False otherwise. 443 """ 444 status = system('make install') 445 return status == 0 446 447 448 def fetch(self, dest_dir): 449 """ 450 Fetch the package from one its URLs and save it in dest_dir. 451 452 If the the package already exists in dest_dir and the checksum 453 matches this code will not fetch it again. 454 455 Sets the 'verified_package' attribute with the destination pathname. 456 457 @param dest_dir - The destination directory to save the local file. 458 If it does not exist it will be created. 459 460 @returns A boolean indicating if we the package is now in dest_dir. 461 @raises FetchError - When something unexpected happens. 462 """ 463 if not os.path.exists(dest_dir): 464 os.makedirs(dest_dir) 465 local_path = os.path.join(dest_dir, self.local_filename) 466 467 # If the package exists, verify its checksum and be happy if it is good. 468 if os.path.exists(local_path): 469 actual_hex_sum = _checksum_file(local_path) 470 if self.hex_sum == actual_hex_sum: 471 logging.info('Good checksum for existing %s package.', 472 self.name) 473 self.verified_package = local_path 474 return True 475 logging.warning('Bad checksum for existing %s package. ' 476 'Re-downloading', self.name) 477 os.rename(local_path, local_path + '.wrong-checksum') 478 479 # Download the package from one of its urls, rejecting any if the 480 # checksum does not match. 481 for url in self.urls: 482 logging.info('Fetching %s', url) 483 try: 484 url_file = urllib2.urlopen(url) 485 except (urllib2.URLError, EnvironmentError): 486 logging.warning('Could not fetch %s package from %s.', 487 self.name, url) 488 continue 489 490 data_length = int(url_file.info().get('Content-Length', 491 _MAX_PACKAGE_SIZE)) 492 if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE: 493 raise FetchError('%s from %s fails Content-Length %d ' 494 'sanity check.' % (self.name, url, 495 data_length)) 496 checksum = utils.hash('sha1') 497 total_read = 0 498 output = open(local_path, 'wb') 499 try: 500 while total_read < data_length: 501 data = url_file.read(_READ_SIZE) 502 if not data: 503 break 504 output.write(data) 505 checksum.update(data) 506 total_read += len(data) 507 finally: 508 output.close() 509 if self.hex_sum != checksum.hexdigest(): 510 logging.warning('Bad checksum for %s fetched from %s.', 511 self.name, url) 512 logging.warning('Got %s', checksum.hexdigest()) 513 logging.warning('Expected %s', self.hex_sum) 514 os.unlink(local_path) 515 continue 516 logging.info('Good checksum.') 517 self.verified_package = local_path 518 return True 519 else: 520 return False 521 522 523# NOTE: This class definition must come -before- all other ExternalPackage 524# classes that need to use this version of setuptools so that is is inserted 525# into the ExternalPackage.subclasses list before them. 526class SetuptoolsPackage(ExternalPackage): 527 """setuptools package""" 528 # For all known setuptools releases a string compare works for the 529 # version string. Hopefully they never release a 0.10. (Their own 530 # version comparison code would break if they did.) 531 # Any system with setuptools > 18.0.1 is fine. If none installed, then 532 # try to install the latest found on the upstream. 533 minimum_version = '18.0.1' 534 version = '18.0.1' 535 urls = ('http://pypi.python.org/packages/source/s/setuptools/' 536 'setuptools-%s.tar.gz' % (version,),) 537 local_filename = 'setuptools-%s.tar.gz' % version 538 hex_sum = 'ebc4fe81b7f6d61d923d9519f589903824044f52' 539 540 SUDO_SLEEP_DELAY = 15 541 542 543 def _build_and_install(self, install_dir): 544 """Install setuptools on the system.""" 545 logging.info('NOTE: setuptools install does not use install_dir.') 546 return self._build_and_install_from_package(install_dir) 547 548 549 def _build_and_install_current_dir(self, install_dir): 550 egg_path = self._build_egg_using_setup_py() 551 if not egg_path: 552 return False 553 554 print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n' 555 print 'About to run sudo to install setuptools', self.version 556 print 'on your system for use by', sys.executable, '\n' 557 print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n' 558 time.sleep(self.SUDO_SLEEP_DELAY) 559 560 # Copy the egg to the local filesystem /var/tmp so that root can 561 # access it properly (avoid NFS squashroot issues). 562 temp_dir = self._get_temp_dir() 563 try: 564 shutil.copy(egg_path, temp_dir) 565 egg_name = os.path.split(egg_path)[1] 566 temp_egg = os.path.join(temp_dir, egg_name) 567 p = subprocess.Popen(['sudo', '/bin/sh', temp_egg], 568 stdout=subprocess.PIPE) 569 regex = re.compile('Copying (.*?) to (.*?)\n') 570 match = regex.search(p.communicate()[0]) 571 status = p.wait() 572 573 if match: 574 compiled = os.path.join(match.group(2), match.group(1)) 575 os.system("sudo chmod a+r '%s'" % compiled) 576 finally: 577 shutil.rmtree(temp_dir) 578 579 if status: 580 logging.error('install of setuptools from egg failed.') 581 return False 582 return True 583 584 585class MySQLdbPackage(ExternalPackage): 586 """mysql package, used in scheduler.""" 587 module_name = 'MySQLdb' 588 version = '1.2.3' 589 urls = ('http://downloads.sourceforge.net/project/mysql-python/' 590 'mysql-python/%(version)s/MySQL-python-%(version)s.tar.gz' 591 % dict(version=version),) 592 local_filename = 'MySQL-python-%s.tar.gz' % version 593 hex_sum = '3511bb8c57c6016eeafa531d5c3ea4b548915e3c' 594 595 _build_and_install_current_dir = ( 596 ExternalPackage._build_and_install_current_dir_setup_py) 597 598 599 def _build_and_install(self, install_dir): 600 if not os.path.exists('/usr/bin/mysql_config'): 601 error_msg = ('You need to install /usr/bin/mysql_config.\n' 602 'On Ubuntu or Debian based systems use this: ' 603 'sudo apt-get install libmysqlclient15-dev') 604 logging.error(error_msg) 605 return False, error_msg 606 return self._build_and_install_from_package(install_dir) 607 608 609class DjangoPackage(ExternalPackage): 610 """django package.""" 611 version = '1.5.1' 612 local_filename = 'Django-%s.tar.gz' % version 613 urls = ('http://www.djangoproject.com/download/%s/tarball/' % version,) 614 hex_sum = '0ab97b90c4c79636e56337f426f1e875faccbba1' 615 616 _build_and_install = ExternalPackage._build_and_install_from_package 617 _build_and_install_current_dir = ( 618 ExternalPackage._build_and_install_current_dir_noegg) 619 620 621 def _get_installed_version_from_module(self, module): 622 try: 623 return module.get_version().split()[0] 624 except AttributeError: 625 return '0.9.6' 626 627 628 629class NumpyPackage(ExternalPackage): 630 """numpy package, required by matploglib.""" 631 version = '1.7.0' 632 local_filename = 'numpy-%s.tar.gz' % version 633 urls = ('http://downloads.sourceforge.net/project/numpy/NumPy/%(version)s/' 634 'numpy-%(version)s.tar.gz' % dict(version=version),) 635 hex_sum = 'ba328985f20390b0f969a5be2a6e1141d5752cf9' 636 637 _build_and_install = ExternalPackage._build_and_install_from_package 638 _build_and_install_current_dir = ( 639 ExternalPackage._build_and_install_current_dir_setupegg_py) 640 641 642class MatplotlibPackage(ExternalPackage): 643 """ 644 matplotlib package 645 646 This requires numpy so it must be declared after numpy to guarantee that 647 it is already installed. 648 """ 649 version = '0.98.5.3' 650 short_version = '0.98.5' 651 local_filename = 'matplotlib-%s.tar.gz' % version 652 urls = ('http://downloads.sourceforge.net/project/matplotlib/matplotlib/' 653 'matplotlib-%s/matplotlib-%s.tar.gz' % (short_version, version),) 654 hex_sum = '2f6c894cf407192b3b60351bcc6468c0385d47b6' 655 os_requirements = {('/usr/include/freetype2/ft2build.h', 656 '/usr/include/ft2build.h'): 'libfreetype6-dev', 657 ('/usr/include/png.h'): 'libpng12-dev'} 658 659 _build_and_install = ExternalPackage._build_and_install_from_package 660 _build_and_install_current_dir = ( 661 ExternalPackage._build_and_install_current_dir_setupegg_py) 662 663 664class AtForkPackage(ExternalPackage): 665 """atfork package""" 666 version = '0.1.2' 667 local_filename = 'atfork-%s.zip' % version 668 urls = ('http://python-atfork.googlecode.com/files/' + local_filename,) 669 hex_sum = '5baa64c73e966b57fa797040585c760c502dc70b' 670 671 _build_and_install = ExternalPackage._build_and_install_from_package 672 _build_and_install_current_dir = ( 673 ExternalPackage._build_and_install_current_dir_noegg) 674 675 676class ParamikoPackage(ExternalPackage): 677 """paramiko package""" 678 version = '1.7.5' 679 local_filename = 'paramiko-%s.zip' % version 680 urls = ('https://pypi.python.org/packages/source/p/paramiko/' + local_filename,) 681 hex_sum = 'd23e437c0d8bd6aeb181d9990a9d670fb30d0c72' 682 683 684 _build_and_install = ExternalPackage._build_and_install_from_package 685 686 687 def _check_for_pycrypto(self): 688 # NOTE(gps): Linux distros have better python-crypto packages than we 689 # can easily get today via a wget due to the library's age and staleness 690 # yet many security and behavior bugs are fixed by patches that distros 691 # already apply. PyCrypto has a new active maintainer in 2009. Once a 692 # new release is made (http://pycrypto.org/) we should add an installer. 693 try: 694 import Crypto 695 except ImportError: 696 logging.error('Please run "sudo apt-get install python-crypto" ' 697 'or your Linux distro\'s equivalent.') 698 return False 699 return True 700 701 702 def _build_and_install_current_dir(self, install_dir): 703 if not self._check_for_pycrypto(): 704 return False 705 # paramiko 1.7.4 doesn't require building, it is just a module directory 706 # that we can rsync into place directly. 707 if not os.path.isdir('paramiko'): 708 raise Error('no paramiko directory in %s.' % os.getcwd()) 709 status = system("rsync -r 'paramiko' '%s/'" % install_dir) 710 if status: 711 logging.error('%s rsync to install_dir failed.', self.name) 712 return False 713 return True 714 715 716class RequestsPackage(ExternalPackage): 717 """requests package""" 718 version = '0.11.2' 719 local_filename = 'requests-%s.tar.gz' % version 720 urls = ('http://pypi.python.org/packages/source/r/requests/' + 721 local_filename,) 722 hex_sum = '00a49e8bd6dd8955acf6f6269d1b85f50c70b712' 723 724 _build_and_install = ExternalPackage._build_and_install_from_package 725 _build_and_install_current_dir = ( 726 ExternalPackage._build_and_install_current_dir_setup_py) 727 728 729class JsonRPCLib(ExternalPackage): 730 """jsonrpclib package""" 731 version = '0.1.3' 732 module_name = 'jsonrpclib' 733 local_filename = '%s-%s.tar.gz' % (module_name, version) 734 urls = ('http://pypi.python.org/packages/source/j/%s/%s' % 735 (module_name, local_filename), ) 736 hex_sum = '431714ed19ab677f641ce5d678a6a95016f5c452' 737 738 def _get_installed_version_from_module(self, module): 739 # jsonrpclib doesn't contain a proper version 740 return self.version 741 742 _build_and_install = ExternalPackage._build_and_install_from_package 743 _build_and_install_current_dir = ( 744 ExternalPackage._build_and_install_current_dir_noegg) 745 746 747class Httplib2Package(ExternalPackage): 748 """httplib2 package""" 749 version = '0.6.0' 750 local_filename = 'httplib2-%s.tar.gz' % version 751 urls = ('http://httplib2.googlecode.com/files/' + local_filename,) 752 hex_sum = '995344b2704826cc0d61a266e995b328d92445a5' 753 754 def _get_installed_version_from_module(self, module): 755 # httplib2 doesn't contain a proper version 756 return self.version 757 758 _build_and_install = ExternalPackage._build_and_install_from_package 759 _build_and_install_current_dir = ( 760 ExternalPackage._build_and_install_current_dir_noegg) 761 762 763class GwtPackage(ExternalPackage): 764 """Fetch and extract a local copy of GWT used to build the frontend.""" 765 766 version = '2.3.0' 767 local_filename = 'gwt-%s.zip' % version 768 urls = ('http://google-web-toolkit.googlecode.com/files/' + local_filename,) 769 hex_sum = 'd51fce9166e6b31349659ffca89baf93e39bc84b' 770 name = 'gwt' 771 about_filename = 'about.txt' 772 module_name = None # Not a Python module. 773 774 775 def is_needed(self, install_dir): 776 gwt_dir = os.path.join(install_dir, self.name) 777 about_file = os.path.join(install_dir, self.name, self.about_filename) 778 779 if not os.path.exists(gwt_dir) or not os.path.exists(about_file): 780 logging.info('gwt not installed for autotest') 781 return True 782 783 f = open(about_file, 'r') 784 version_line = f.readline() 785 f.close() 786 787 match = re.match(r'Google Web Toolkit (.*)', version_line) 788 if not match: 789 logging.info('did not find gwt version') 790 return True 791 792 logging.info('found gwt version %s', match.group(1)) 793 return match.group(1) != self.version 794 795 796 def _build_and_install(self, install_dir): 797 os.chdir(install_dir) 798 self._extract_compressed_package() 799 extracted_dir = self.local_filename[:-len('.zip')] 800 target_dir = os.path.join(install_dir, self.name) 801 if os.path.exists(target_dir): 802 shutil.rmtree(target_dir) 803 os.rename(extracted_dir, target_dir) 804 return True 805 806 807class GVizAPIPackage(ExternalPackage): 808 """gviz package""" 809 module_name = 'gviz_api' 810 version = '1.7.0' 811 url_filename = 'gviz_api_py-%s.tar.gz' % version 812 local_filename = 'google-visualization-python.tar.gz' 813 urls = ('http://google-visualization-python.googlecode.com/files/%s' % ( 814 url_filename),) 815 hex_sum = 'cd9a0fb4ca5c4f86c0d85756f501fd54ccf492d2' 816 817 _build_and_install = ExternalPackage._build_and_install_from_package 818 _build_and_install_current_dir = ( 819 ExternalPackage._build_and_install_current_dir_noegg) 820 821 def _get_installed_version_from_module(self, module): 822 # gviz doesn't contain a proper version 823 return self.version 824 825 826class StatsdPackage(ExternalPackage): 827 """python-statsd package""" 828 version = '1.7.2' 829 url_filename = 'python-statsd-%s.tar.gz' % version 830 local_filename = url_filename 831 urls = ('http://pypi.python.org/packages/source/p/python-statsd/%s' % ( 832 url_filename),) 833 hex_sum = '2cc186ebdb723e2420b432ab71639786d877694b' 834 835 _build_and_install = ExternalPackage._build_and_install_from_package 836 _build_and_install_current_dir = ( 837 ExternalPackage._build_and_install_current_dir_setup_py) 838 839 840class GdataPackage(ExternalPackage): 841 """ 842 Pulls the GData library, giving us an API to query tracker. 843 """ 844 845 version = '2.0.14' 846 url_filename = 'gdata-%s.tar.gz' % version 847 local_filename = url_filename 848 urls = ('http://gdata-python-client.googlecode.com/files/%s' % ( 849 url_filename),) 850 hex_sum = '5eed0e01ab931e3f706ec544fc8f06ecac384e91' 851 852 _build_and_install = ExternalPackage._build_and_install_from_package 853 _build_and_install_current_dir = ( 854 ExternalPackage._build_and_install_current_dir_noegg) 855 856 def _get_installed_version_from_module(self, module): 857 # gdata doesn't contain a proper version 858 return self.version 859 860 861class GoogleAPIClientPackage(ExternalPackage): 862 """ 863 Pulls the Python Google API client library. 864 """ 865 version = '1.1' 866 module_name = 'apiclient' 867 url_filename = 'google-api-python-client-%s.tar.gz' % version 868 local_filename = url_filename 869 urls = ('https://google-api-python-client.googlecode.com/files/%s' % ( 870 url_filename),) 871 hex_sum = '2294949683e367b3d4ecaeb77502509c5af21e60' 872 873 _build_and_install = ExternalPackage._build_and_install_from_package 874 _build_and_install_current_dir = ( 875 ExternalPackage._build_and_install_current_dir_setup_py) 876 877 878class GFlagsPackage(ExternalPackage): 879 """ 880 Gets the Python GFlags client library. 881 """ 882 # gflags doesn't contain a proper version 883 version = '2.0' 884 url_filename = 'python-gflags-%s.tar.gz' % version 885 local_filename = url_filename 886 urls = ('https://python-gflags.googlecode.com/files/%s' % ( 887 url_filename),) 888 hex_sum = 'db309e6964b102ff36de319ce551db512a78281e' 889 890 _build_and_install = ExternalPackage._build_and_install_from_package 891 _build_and_install_current_dir = ( 892 ExternalPackage._build_and_install_current_dir_setup_py) 893 894 895 def _get_installed_version_from_module(self, module): 896 return self.version 897 898 899class DnsPythonPackage(ExternalPackage): 900 """ 901 dns module 902 903 Used in unittests. 904 """ 905 module_name = 'dns' 906 version = '1.3.5' 907 url_filename = 'dnspython-%s.tar.gz' % version 908 local_filename = url_filename 909 urls = ('http://www.dnspython.org/kits/%s/%s' % ( 910 version, url_filename),) 911 912 hex_sum = '06314dad339549613435470c6add992910e26e5d' 913 914 _build_and_install = ExternalPackage._build_and_install_from_package 915 _build_and_install_current_dir = ( 916 ExternalPackage._build_and_install_current_dir_noegg) 917 918 def _get_installed_version_from_module(self, module): 919 """Ask our module its version string and return it or '' if unknown.""" 920 try: 921 __import__(self.module_name + '.version') 922 return module.version.version 923 except AttributeError: 924 logging.error('could not get version from %s', module) 925 return '' 926 927 928class PyudevPackage(ExternalPackage): 929 """ 930 pyudev module 931 932 Used in unittests. 933 """ 934 version = '0.16.1' 935 url_filename = 'pyudev-%s.tar.gz' % version 936 local_filename = url_filename 937 urls = ('http://pypi.python.org/packages/source/p/pyudev/%s' % ( 938 url_filename),) 939 hex_sum = 'b36bc5c553ce9b56d32a5e45063a2c88156771c0' 940 941 _build_and_install = ExternalPackage._build_and_install_from_package 942 _build_and_install_current_dir = ( 943 ExternalPackage._build_and_install_current_dir_setup_py) 944 945 946class PyMoxPackage(ExternalPackage): 947 """ 948 mox module 949 950 Used in unittests. 951 """ 952 module_name = 'mox' 953 version = '0.5.3' 954 url_filename = 'mox-%s.tar.gz' % version 955 local_filename = url_filename 956 urls = ('http://pypi.python.org/packages/source/m/mox/%s' % ( 957 url_filename),) 958 hex_sum = '1c502d2c0a8aefbba2c7f385a83d33e7d822452a' 959 960 _build_and_install = ExternalPackage._build_and_install_from_package 961 _build_and_install_current_dir = ( 962 ExternalPackage._build_and_install_current_dir_noegg) 963 964 def _get_installed_version_from_module(self, module): 965 # mox doesn't contain a proper version 966 return self.version 967 968 969class PySeleniumPackage(ExternalPackage): 970 """ 971 selenium module 972 973 Used in wifi_interop suite. 974 """ 975 module_name = 'selenium' 976 version = '2.37.2' 977 url_filename = 'selenium-%s.tar.gz' % version 978 local_filename = url_filename 979 urls = ('https://pypi.python.org/packages/source/s/selenium/%s' % ( 980 url_filename),) 981 hex_sum = '66946d5349e36d946daaad625c83c30c11609e36' 982 983 _build_and_install = ExternalPackage._build_and_install_from_package 984 _build_and_install_current_dir = ( 985 ExternalPackage._build_and_install_current_dir_setup_py) 986 987 988class FaultHandlerPackage(ExternalPackage): 989 """ 990 faulthandler module 991 """ 992 module_name = 'faulthandler' 993 version = '2.3' 994 url_filename = '%s-%s.tar.gz' % (module_name, version) 995 local_filename = url_filename 996 urls = ('http://pypi.python.org/packages/source/f/faulthandler/%s' % 997 (url_filename),) 998 hex_sum = 'efb30c068414fba9df892e48fcf86170cbf53589' 999 1000 _build_and_install = ExternalPackage._build_and_install_from_package 1001 _build_and_install_current_dir = ( 1002 ExternalPackage._build_and_install_current_dir_noegg) 1003 1004 1005class PsutilPackage(ExternalPackage): 1006 """ 1007 psutil module 1008 """ 1009 module_name = 'psutil' 1010 version = '2.1.1' 1011 url_filename = '%s-%s.tar.gz' % (module_name, version) 1012 local_filename = url_filename 1013 urls = ('http://pypi.python.org/packages/source/p/psutil/%s' % 1014 (url_filename),) 1015 hex_sum = '0c20a20ed316e69f2b0881530439213988229916' 1016 1017 _build_and_install = ExternalPackage._build_and_install_from_package 1018 _build_and_install_current_dir = ( 1019 ExternalPackage._build_and_install_current_dir_setup_py) 1020 1021 1022class ElasticSearchPackage(ExternalPackage): 1023 """elasticsearch-py package.""" 1024 version = '1.6.0' 1025 url_filename = 'elasticsearch-%s.tar.gz' % version 1026 local_filename = url_filename 1027 urls = ('https://pypi.python.org/packages/source/e/elasticsearch/%s' % 1028 (url_filename),) 1029 hex_sum = '3e676c96f47935b1f52df82df3969564bd356b1c' 1030 _build_and_install = ExternalPackage._build_and_install_from_package 1031 _build_and_install_current_dir = ( 1032 ExternalPackage._build_and_install_current_dir_setup_py) 1033 1034 1035class Urllib3Package(ExternalPackage): 1036 """elasticsearch-py package.""" 1037 version = '1.9' 1038 url_filename = 'urllib3-%s.tar.gz' % version 1039 local_filename = url_filename 1040 urls = ('https://pypi.python.org/packages/source/u/urllib3/%s' % 1041 (url_filename),) 1042 hex_sum = '9522197efb2a2b49ce804de3a515f06d97b6602f' 1043 _build_and_install = ExternalPackage._build_and_install_from_package 1044 _build_and_install_current_dir = ( 1045 ExternalPackage._build_and_install_current_dir_setup_py) 1046 1047 1048class ImagingLibraryPackage(ExternalPackage): 1049 """Python Imaging Library (PIL).""" 1050 version = '1.1.7' 1051 url_filename = 'Imaging-%s.tar.gz' % version 1052 local_filename = url_filename 1053 urls = ('http://effbot.org/downloads/%s' % url_filename,) 1054 hex_sum = '76c37504251171fda8da8e63ecb8bc42a69a5c81' 1055 1056 def _build_and_install(self, install_dir): 1057 # The path of zlib library might be different from what PIL setup.py is 1058 # expected. Following change does the best attempt to link the library 1059 # to a path PIL setup.py will try. 1060 libz_possible_path = '/usr/lib/x86_64-linux-gnu/libz.so' 1061 libz_expected_path = '/usr/lib/libz.so' 1062 if (os.path.exists(libz_possible_path) and 1063 not os.path.exists(libz_expected_path)): 1064 utils.run('sudo ln -s %s %s' % 1065 (libz_possible_path, libz_expected_path)) 1066 return self._build_and_install_from_package(install_dir) 1067 1068 _build_and_install_current_dir = ( 1069 ExternalPackage._build_and_install_current_dir_noegg) 1070 1071 1072class _ExternalGitRepo(ExternalPackage): 1073 """ 1074 Parent class for any package which needs to pull a git repo. 1075 1076 This class inherits from ExternalPackage only so we can sync git 1077 repos through the build_externals script. We do not reuse any of 1078 ExternalPackage's other methods. Any package that needs a git repo 1079 should subclass this and override build_and_install or fetch as 1080 they see appropriate. 1081 """ 1082 1083 os_requirements = {('/usr/bin/git') : 'git-core'} 1084 1085 # All the chromiumos projects used on the lab servers should have a 'prod' 1086 # branch used to track the software version deployed in prod. 1087 PROD_BRANCH = 'prod' 1088 1089 def is_needed(self, unused_install_dir): 1090 """Tell build_externals that we need to fetch.""" 1091 # TODO(beeps): check if we're already upto date. 1092 return True 1093 1094 1095 def build_and_install(self, unused_install_dir): 1096 """ 1097 Fall through method to install a package. 1098 1099 Overwritten in base classes to pull a git repo. 1100 """ 1101 raise NotImplementedError 1102 1103 1104 def fetch(self, unused_dest_dir): 1105 """Fallthrough method to fetch a package.""" 1106 return True 1107 1108 1109class HdctoolsRepo(_ExternalGitRepo): 1110 """Clones or updates the hdctools repo.""" 1111 1112 module_name = 'servo' 1113 temp_hdctools_dir = tempfile.mktemp(suffix='hdctools') 1114 _GIT_URL = ('https://chromium.googlesource.com/' 1115 'chromiumos/third_party/hdctools') 1116 1117 def fetch(self, unused_dest_dir): 1118 """ 1119 Fetch repo to a temporary location. 1120 1121 We use an intermediate temp directory to stage our 1122 installation because we only care about the servo package. 1123 If we can't get at the top commit hash after fetching 1124 something is wrong. This can happen when we've cloned/pulled 1125 an empty repo. Not something we expect to do. 1126 1127 @parma unused_dest_dir: passed in because we inherit from 1128 ExternalPackage. 1129 1130 @return: True if repo sync was successful. 1131 """ 1132 git_repo = revision_control.GitRepo( 1133 self.temp_hdctools_dir, 1134 self._GIT_URL, 1135 None, 1136 abs_work_tree=self.temp_hdctools_dir) 1137 git_repo.reinit_repo_at(self.PROD_BRANCH) 1138 1139 if git_repo.get_latest_commit_hash(): 1140 return True 1141 return False 1142 1143 1144 def build_and_install(self, install_dir): 1145 """Reach into the hdctools repo and rsync only the servo directory.""" 1146 1147 servo_dir = os.path.join(self.temp_hdctools_dir, 'servo') 1148 if not os.path.exists(servo_dir): 1149 return False 1150 1151 rv = self._rsync(servo_dir, os.path.join(install_dir, 'servo')) 1152 shutil.rmtree(self.temp_hdctools_dir) 1153 return rv 1154 1155 1156class ChromiteRepo(_ExternalGitRepo): 1157 """Clones or updates the chromite repo.""" 1158 1159 _GIT_URL = ('https://chromium.googlesource.com/chromiumos/chromite') 1160 1161 def build_and_install(self, install_dir): 1162 """ 1163 Clone if the repo isn't initialized, pull clean bits if it is. 1164 1165 Unlike it's hdctools counterpart the chromite repo clones master 1166 directly into site-packages. It doesn't use an intermediate temp 1167 directory because it doesn't need installation. 1168 1169 @param install_dir: destination directory for chromite installation. 1170 """ 1171 local_chromite_dir = os.path.join(install_dir, 'chromite') 1172 git_repo = revision_control.GitRepo( 1173 local_chromite_dir, 1174 self._GIT_URL, 1175 abs_work_tree=local_chromite_dir) 1176 git_repo.reinit_repo_at(self.PROD_BRANCH) 1177 1178 1179 if git_repo.get_latest_commit_hash(): 1180 return True 1181 return False 1182 1183 1184class DevServerRepo(_ExternalGitRepo): 1185 """Clones or updates the chromite repo.""" 1186 1187 _GIT_URL = ('https://chromium.googlesource.com/' 1188 'chromiumos/platform/dev-util') 1189 1190 def build_and_install(self, install_dir): 1191 """ 1192 Clone if the repo isn't initialized, pull clean bits if it is. 1193 1194 Unlike it's hdctools counterpart the dev-util repo clones master 1195 directly into site-packages. It doesn't use an intermediate temp 1196 directory because it doesn't need installation. 1197 1198 @param install_dir: destination directory for chromite installation. 1199 """ 1200 local_devserver_dir = os.path.join(install_dir, 'devserver') 1201 git_repo = revision_control.GitRepo(local_devserver_dir, self._GIT_URL, 1202 abs_work_tree=local_devserver_dir) 1203 git_repo.reinit_repo_at(self.PROD_BRANCH) 1204 1205 if git_repo.get_latest_commit_hash(): 1206 return True 1207 return False 1208 1209 1210class BtsocketRepo(_ExternalGitRepo): 1211 """Clones or updates the btsocket repo.""" 1212 1213 _GIT_URL = ('https://chromium.googlesource.com/' 1214 'chromiumos/platform/btsocket') 1215 1216 def fetch(self, unused_dest_dir): 1217 """ 1218 Fetch repo to a temporary location. 1219 1220 We use an intermediate temp directory because we have to build an 1221 egg for installation. If we can't get at the top commit hash after 1222 fetching something is wrong. This can happen when we've cloned/pulled 1223 an empty repo. Not something we expect to do. 1224 1225 @parma unused_dest_dir: passed in because we inherit from 1226 ExternalPackage. 1227 1228 @return: True if repo sync was successful. 1229 """ 1230 self.temp_btsocket_dir = autotemp.tempdir(unique_id='btsocket') 1231 try: 1232 git_repo = revision_control.GitRepo( 1233 self.temp_btsocket_dir.name, 1234 self._GIT_URL, 1235 None, 1236 abs_work_tree=self.temp_btsocket_dir.name) 1237 git_repo.reinit_repo_at(self.PROD_BRANCH) 1238 1239 if git_repo.get_latest_commit_hash(): 1240 return True 1241 except: 1242 self.temp_btsocket_dir.clean() 1243 raise 1244 1245 self.temp_btsocket_dir.clean() 1246 return False 1247 1248 1249 def build_and_install(self, install_dir): 1250 """ 1251 Install the btsocket module using setup.py 1252 1253 @param install_dir: Target installation directory. 1254 1255 @return: A boolean indicating success of failure. 1256 """ 1257 work_dir = os.getcwd() 1258 try: 1259 os.chdir(self.temp_btsocket_dir.name) 1260 rv = self._build_and_install_current_dir_setup_py(install_dir) 1261 finally: 1262 os.chdir(work_dir) 1263 self.temp_btsocket_dir.clean() 1264 return rv 1265