1#!/usr/bin/python
2#
3# Please keep this code python 2.4 compatible and stand alone.
4
5"""
6Fetch, build and install external Python library dependancies.
7
8This fetches external python libraries, builds them using your host's
9python and installs them under our own autotest/site-packages/ directory.
10
11Usage?  Just run it.
12    utils/build_externals.py
13"""
14
15import compileall, logging, os, sys
16import common
17from autotest_lib.client.common_lib import logging_config, logging_manager
18from autotest_lib.client.common_lib import utils
19from autotest_lib.utils import external_packages
20
21# bring in site packages as well
22utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages')
23
24# Where package source be fetched to relative to the top of the autotest tree.
25PACKAGE_DIR = 'ExternalSource'
26
27# Where packages will be installed to relative to the top of the autotest tree.
28INSTALL_DIR = 'site-packages'
29
30# Installs all packages, even if the system already has the version required
31INSTALL_ALL = False
32
33
34# Want to add more packages to fetch, build and install?  See the class
35# definitions at the end of external_packages.py for examples of how to do it.
36
37
38class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
39    def configure_logging(self, results_dir=None, verbose=False):
40        super(BuildExternalsLoggingConfig, self).configure_logging(
41                                                               use_console=True,
42                                                               verbose=verbose)
43
44
45def main():
46    """
47    Find all ExternalPackage classes defined in this file and ask them to
48    fetch, build and install themselves.
49    """
50    logging_manager.configure_logging(BuildExternalsLoggingConfig(),
51                                      verbose=True)
52    os.umask(022)
53
54    top_of_tree = external_packages.find_top_of_autotest_tree()
55    package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
56    install_dir = os.path.join(top_of_tree, INSTALL_DIR)
57
58    # Make sure the install_dir is in our python module search path
59    # as well as the PYTHONPATH being used by all our setup.py
60    # install subprocesses.
61    if install_dir not in sys.path:
62        sys.path.insert(0, install_dir)
63    env_python_path_varname = 'PYTHONPATH'
64    env_python_path = os.environ.get(env_python_path_varname, '')
65    if install_dir+':' not in env_python_path:
66        os.environ[env_python_path_varname] = ':'.join([
67            install_dir, env_python_path])
68
69    fetched_packages, fetch_errors = fetch_necessary_packages(package_dir,
70                                                              install_dir)
71    install_errors = build_and_install_packages(fetched_packages, install_dir)
72
73    # Byte compile the code after it has been installed in its final
74    # location as .pyc files contain the path passed to compile_dir().
75    # When printing exception tracebacks, python uses that path first to look
76    # for the source code before checking the directory of the .pyc file.
77    # Don't leave references to our temporary build dir in the files.
78    logging.info('compiling .py files in %s to .pyc', install_dir)
79    compileall.compile_dir(install_dir, quiet=True)
80
81    # Some things install with whacky permissions, fix that.
82    external_packages.system("chmod -R a+rX '%s'" % install_dir)
83
84    errors = fetch_errors + install_errors
85    for error_msg in errors:
86        logging.error(error_msg)
87
88    return len(errors)
89
90
91def fetch_necessary_packages(dest_dir, install_dir):
92    """
93    Fetches all ExternalPackages into dest_dir.
94
95    @param dest_dir: Directory the packages should be fetched into.
96    @param install_dir: Directory where packages will later installed.
97
98    @returns A tuple containing two lists:
99             * A list of ExternalPackage instances that were fetched and
100               need to be installed.
101             * A list of error messages for any failed fetches.
102    """
103    names_to_check = sys.argv[1:]
104    errors = []
105    fetched_packages = []
106    for package_class in external_packages.ExternalPackage.subclasses:
107        package = package_class()
108        if names_to_check and package.name.lower() not in names_to_check:
109            continue
110        if not package.is_needed(install_dir):
111            logging.info('A new %s is not needed on this system.',
112                         package.name)
113            if INSTALL_ALL:
114                logging.info('Installing anyways...')
115            else:
116                continue
117        if not package.fetch(dest_dir):
118            msg = 'Unable to download %s' % package.name
119            logging.error(msg)
120            errors.append(msg)
121        else:
122            fetched_packages.append(package)
123
124    return fetched_packages, errors
125
126
127def build_and_install_packages(packages, install_dir):
128    """
129    Builds and installs all packages into install_dir.
130
131    @param packages - A list of already fetched ExternalPackage instances.
132    @param install_dir - Directory the packages should be installed into.
133
134    @returns A list of error messages for any installs that failed.
135    """
136    errors = []
137    for package in packages:
138        result = package.build_and_install(install_dir)
139        if isinstance(result, bool):
140            success = result
141            message = None
142        else:
143            success = result[0]
144            message = result[1]
145        if not success:
146            msg = ('Unable to build and install %s.\nError: %s' %
147                   (package.name, message))
148            logging.error(msg)
149            errors.append(msg)
150    return errors
151
152
153if __name__ == '__main__':
154    sys.exit(main())
155