#pylint: disable-msg=C0111 """ This module defines the BasePackageManager Class which provides an implementation of the packaging system API providing methods to fetch, upload and remove packages. Site specific extensions to any of these methods should inherit this class. """ import fcntl, logging, os, re, shutil from autotest_lib.client.bin import os_dep from autotest_lib.client.common_lib import error, utils, global_config # the name of the checksum file that stores the packages' checksums CHECKSUM_FILE = "packages.checksum" def has_pbzip2(): '''Check if parallel bzip2 is available on this system.''' try: os_dep.command('pbzip2') except ValueError: return False return True # is parallel bzip2 available for use? _PBZIP2_AVAILABLE = has_pbzip2() def parse_ssh_path(repo): ''' Parse ssh://xx@xx/path/to/ and return a tuple with host_line and remote path ''' match = re.search('^ssh://(.*?)(/.*)$', repo) if match: return match.groups() else: raise error.PackageUploadError( "Incorrect SSH path in global_config: %s" % repo) def repo_run_command(repo, cmd, ignore_status=False, cd=True): """Run a command relative to the repos path""" repo = repo.strip() run_cmd = None cd_str = '' if repo.startswith('ssh://'): username = None hostline, remote_path = parse_ssh_path(repo) if cd: cd_str = 'cd %s && ' % remote_path if '@' in hostline: username, host = hostline.split('@') run_cmd = 'ssh %s@%s "%s%s"' % (username, host, cd_str, cmd) else: run_cmd = 'ssh %s "%s%s"' % (host, cd_str, cmd) else: if cd: cd_str = 'cd %s && ' % repo run_cmd = "%s%s" % (cd_str, cmd) if run_cmd: return utils.run(run_cmd, ignore_status=ignore_status) def create_directory(repo): remote_path = repo if repo.startswith('ssh://'): _, remote_path = parse_ssh_path(repo) repo_run_command(repo, 'mkdir -p %s' % remote_path, cd=False) def check_diskspace(repo, min_free=None): # Note: 1 GB = 10**9 bytes (SI unit). if min_free is None: min_free = global_config.global_config.get_config_value('PACKAGES', 'minimum_free_space', type=int, default=1) try: df = repo_run_command(repo, 'df -PB %d . | tail -1' % 10 ** 9).stdout.split() free_space_gb = int(df[3]) except Exception, e: raise error.RepoUnknownError('Unknown Repo Error: %s' % e) if free_space_gb < min_free: raise error.RepoDiskFullError('Not enough disk space available ' '%sg < %sg' % (free_space_gb, min_free)) def check_write(repo): try: repo_testfile = '.repo_test_file' repo_run_command(repo, 'touch %s' % repo_testfile).stdout.strip() repo_run_command(repo, 'rm ' + repo_testfile) except error.CmdError: raise error.RepoWriteError('Unable to write to ' + repo) def trim_custom_directories(repo, older_than_days=None): if not repo: return if older_than_days is None: older_than_days = global_config.global_config.get_config_value( 'PACKAGES', 'custom_max_age', type=int, default=40) cmd = 'find . -type f -atime +%s -exec rm -f {} \;' % older_than_days repo_run_command(repo, cmd, ignore_status=True) class RepositoryFetcher(object): url = None def fetch_pkg_file(self, filename, dest_path): """ Fetch a package file from a package repository. @param filename: The filename of the package file to fetch. @param dest_path: Destination path to download the file to. @raises PackageFetchError if the fetch failed """ raise NotImplementedError() class HttpFetcher(RepositoryFetcher): wget_cmd_pattern = 'wget --connect-timeout=15 -nv %s -O %s' def __init__(self, package_manager, repository_url): """ @param repository_url: The base URL of the http repository """ self.run_command = package_manager._run_command self.url = repository_url def _quick_http_test(self): """ Run a simple 30 second wget on the repository to see if it is reachable. This avoids the need to wait for a full 10min timeout. """ # just make a temp file to write a test fetch into mktemp = 'mktemp -u /tmp/tmp.XXXXXX' dest_file_path = self.run_command(mktemp).stdout.strip() try: # build up a wget command http_cmd = self.wget_cmd_pattern % (self.url, dest_file_path) try: self.run_command(http_cmd, _run_command_dargs={'timeout': 30}) except Exception, e: msg = 'HTTP test failed, unable to contact %s: %s' raise error.PackageFetchError(msg % (self.url, e)) finally: self.run_command('rm -rf %s' % dest_file_path) def fetch_pkg_file(self, filename, dest_path): logging.info('Fetching %s from %s to %s', filename, self.url, dest_path) # do a quick test to verify the repo is reachable self._quick_http_test() # try to retrieve the package via http package_url = os.path.join(self.url, filename) try: cmd = self.wget_cmd_pattern % (package_url, dest_path) result = self.run_command(cmd, _run_command_dargs={'timeout': 1200}) file_exists = self.run_command( 'ls %s' % dest_path, _run_command_dargs={'ignore_status': True}).exit_status == 0 if not file_exists: logging.error('wget failed: %s', result) raise error.CmdError(cmd, result) logging.debug('Successfully fetched %s from %s', filename, package_url) except error.CmdError as e: # remove whatever junk was retrieved when the get failed self.run_command('rm -f %s' % dest_path) raise error.PackageFetchError('%s not found in %s\n%s' 'wget error code: %d' % (filename, package_url, e.result_obj.stderr, e.result_obj.exit_status)) class LocalFilesystemFetcher(RepositoryFetcher): def __init__(self, package_manager, local_dir): self.run_command = package_manager._run_command self.url = local_dir def fetch_pkg_file(self, filename, dest_path): logging.info('Fetching %s from %s to %s', filename, self.url, dest_path) local_path = os.path.join(self.url, filename) try: self.run_command('cp %s %s' % (local_path, dest_path)) logging.debug('Successfully fetched %s from %s', filename, local_path) except error.CmdError, e: raise error.PackageFetchError( 'Package %s could not be fetched from %s' % (filename, self.url), e) class BasePackageManager(object): def __init__(self, pkgmgr_dir, hostname=None, repo_urls=None, upload_paths=None, do_locking=True, run_function=utils.run, run_function_args=[], run_function_dargs={}): ''' repo_urls: The list of the repository urls which is consulted whilst fetching the package upload_paths: The list of the upload of repositories to which the package is uploaded to pkgmgr_dir : A directory that can be used by the package manager to dump stuff (like checksum files of the repositories etc.). do_locking : Enable locking when the packages are installed. run_function is used to execute the commands throughout this file. It defaults to utils.run() but a custom method (if provided) should be of the same schema as utils.run. It should return a CmdResult object and throw a CmdError exception. The reason for using a separate function to run the commands is that the same code can be run to fetch a package on the local machine or on a remote machine (in which case ssh_host's run function is passed in for run_function). ''' # In memory dictionary that stores the checksum's of packages self._checksum_dict = {} self.pkgmgr_dir = pkgmgr_dir self.do_locking = do_locking self.hostname = hostname self.repositories = [] # Create an internal function that is a simple wrapper of # run_function and takes in the args and dargs as arguments def _run_command(command, _run_command_args=run_function_args, _run_command_dargs={}): ''' Special internal function that takes in a command as argument and passes it on to run_function (if specified). The _run_command_dargs are merged into run_function_dargs with the former having more precedence than the latter. ''' new_dargs = dict(run_function_dargs) new_dargs.update(_run_command_dargs) # avoid polluting logs with extremely verbose packaging output new_dargs.update({'stdout_tee' : None}) return run_function(command, *_run_command_args, **new_dargs) self._run_command = _run_command # Process the repository URLs if not repo_urls: repo_urls = [] elif hostname: repo_urls = self.get_mirror_list(repo_urls) for url in repo_urls: self.add_repository(url) # Process the upload URLs if not upload_paths: self.upload_paths = [] else: self.upload_paths = list(upload_paths) def add_repository(self, repo): if isinstance(repo, basestring): self.repositories.append(self.get_fetcher(repo)) elif isinstance(repo, RepositoryFetcher): self.repositories.append(repo) else: raise TypeError("repo must be RepositoryFetcher or url string") def get_fetcher(self, url): if url.startswith('http://'): return HttpFetcher(self, url) else: return LocalFilesystemFetcher(self, url) def repo_check(self, repo): ''' Check to make sure the repo is in a sane state: ensure we have at least XX amount of free space Make sure we can write to the repo ''' if not repo.startswith('/') and not repo.startswith('ssh:'): return try: create_directory(repo) check_diskspace(repo) check_write(repo) except (error.RepoWriteError, error.RepoUnknownError, error.RepoDiskFullError), e: raise error.RepoError("ERROR: Repo %s: %s" % (repo, e)) def upkeep(self, custom_repos=None): ''' Clean up custom upload/download areas ''' from autotest_lib.server import subcommand if not custom_repos: # Not all package types necessarily require or allow custom repos try: custom_repos = global_config.global_config.get_config_value( 'PACKAGES', 'custom_upload_location').split(',') except global_config.ConfigError: custom_repos = [] try: custom_download = global_config.global_config.get_config_value( 'PACKAGES', 'custom_download_location') custom_repos += [custom_download] except global_config.ConfigError: pass if not custom_repos: return subcommand.parallel_simple(trim_custom_directories, custom_repos, log=False) def install_pkg(self, name, pkg_type, fetch_dir, install_dir, preserve_install_dir=False, repo_url=None): ''' Remove install_dir if it already exists and then recreate it unless preserve_install_dir is specified as True. Fetch the package into the pkg_dir. Untar the package into install_dir The assumption is that packages are of the form : ..tar.bz2 name : name of the package type : type of the package fetch_dir : The directory into which the package tarball will be fetched to. install_dir : the directory where the package files will be untarred to repo_url : the url of the repository to fetch the package from. ''' # do_locking flag is on by default unless you disable it (typically # in the cases where packages are directly installed from the server # onto the client in which case fcntl stuff wont work as the code # will run on the server in that case.. if self.do_locking: lockfile_name = '.%s-%s-lock' % (name, pkg_type) lockfile = open(os.path.join(self.pkgmgr_dir, lockfile_name), 'w') try: if self.do_locking: fcntl.flock(lockfile, fcntl.LOCK_EX) self._run_command('mkdir -p %s' % fetch_dir) pkg_name = self.get_tarball_name(name, pkg_type) fetch_path = os.path.join(fetch_dir, pkg_name) try: # Fetch the package into fetch_dir self.fetch_pkg(pkg_name, fetch_path, use_checksum=True) # check to see if the install_dir exists and if it does # then check to see if the .checksum file is the latest install_dir_exists = False try: self._run_command("ls %s" % install_dir) install_dir_exists = True except (error.CmdError, error.AutoservRunError): pass if (install_dir_exists and not self.untar_required(fetch_path, install_dir)): return # untar the package into install_dir and # update the checksum in that directory if not preserve_install_dir: # Make sure we clean up the install_dir self._run_command('rm -rf %s' % install_dir) self._run_command('mkdir -p %s' % install_dir) self.untar_pkg(fetch_path, install_dir) except error.PackageFetchError, why: raise error.PackageInstallError( 'Installation of %s(type:%s) failed : %s' % (name, pkg_type, why)) finally: if self.do_locking: fcntl.flock(lockfile, fcntl.LOCK_UN) lockfile.close() def fetch_pkg(self, pkg_name, dest_path, repo_url=None, use_checksum=False): ''' Fetch the package into dest_dir from repo_url. By default repo_url is None and the package is looked in all the repositories specified. Otherwise it fetches it from the specific repo_url. pkg_name : name of the package (ex: test-sleeptest.tar.bz2, dep-gcc.tar.bz2, kernel.1-1.rpm) repo_url : the URL of the repository where the package is located. dest_path : complete path of where the package will be fetched to. use_checksum : This is set to False to fetch the packages.checksum file so that the checksum comparison is bypassed for the checksum file itself. This is used internally by the packaging system. It should be ignored by externals callers of this method who use it fetch custom packages. ''' try: self._run_command("ls %s" % os.path.dirname(dest_path)) except (error.CmdError, error.AutoservRunError): raise error.PackageFetchError("Please provide a valid " "destination: %s " % dest_path) # See if the package was already fetched earlier, if so # the checksums need to be compared and the package is now # fetched only if they differ. pkg_exists = False try: self._run_command("ls %s" % dest_path) pkg_exists = True except (error.CmdError, error.AutoservRunError): pass # if a repository location is explicitly provided, fetch the package # from there and return if repo_url: repositories = [self.get_fetcher(repo_url)] elif self.repositories: repositories = self.repositories else: raise error.PackageFetchError("No repository urls specified") # install the package from the package repos, try the repos in # reverse order, assuming that the 'newest' repos are most desirable for fetcher in reversed(repositories): try: # Fetch the package if it is not there, the checksum does # not match, or checksums are disabled entirely need_to_fetch = ( not use_checksum or not pkg_exists or not self.compare_checksum(dest_path)) if need_to_fetch: fetcher.fetch_pkg_file(pkg_name, dest_path) # update checksum so we won't refetch next time. if use_checksum: self.update_checksum(dest_path) return except (error.PackageFetchError, error.AutoservRunError) as e: # The package could not be found in this repo, continue looking logging.debug(e) repo_url_list = [repo.url for repo in repositories] message = ('%s could not be fetched from any of the repos %s' % (pkg_name, repo_url_list)) logging.error(message) # if we got here then that means the package is not found # in any of the repositories. raise error.PackageFetchError(message) def upload_pkg(self, pkg_path, upload_path=None, update_checksum=False, timeout=300): from autotest_lib.server import subcommand if upload_path: upload_path_list = [upload_path] self.upkeep(upload_path_list) elif len(self.upload_paths) > 0: self.upkeep() upload_path_list = self.upload_paths else: raise error.PackageUploadError("Invalid Upload Path specified") if update_checksum: # get the packages' checksum file and update it with the current # package's checksum self.update_checksum(pkg_path) commands = [] for path in upload_path_list: commands.append(subcommand.subcommand(self.upload_pkg_parallel, (pkg_path, path, update_checksum))) results = subcommand.parallel(commands, timeout, return_results=True) for result in results: if result: print str(result) # TODO(aganti): Fix the bug with the current checksum logic where # packages' checksums that are not present consistently in all the # repositories are not handled properly. This is a corner case though # but the ideal solution is to make the checksum file repository specific # and then maintain it. def upload_pkg_parallel(self, pkg_path, upload_path, update_checksum=False): ''' Uploads to a specified upload_path or to all the repos. Also uploads the checksum file to all the repos. pkg_path : The complete path to the package file upload_path : the absolute path where the files are copied to. if set to 'None' assumes 'all' repos update_checksum : If set to False, the checksum file is not going to be updated which happens by default. This is necessary for custom packages (like custom kernels and custom tests) that get uploaded which do not need to be part of the checksum file and bloat it. ''' self.repo_check(upload_path) # upload the package if os.path.isdir(pkg_path): self.upload_pkg_dir(pkg_path, upload_path) else: self.upload_pkg_file(pkg_path, upload_path) if update_checksum: self.upload_pkg_file(self._get_checksum_file_path(), upload_path) def upload_pkg_file(self, file_path, upload_path): ''' Upload a single file. Depending on the upload path, the appropriate method for that protocol is called. Currently this simply copies the file to the target directory (but can be extended for other protocols) This assumes that the web server is running on the same machine where the method is being called from. The upload_path's files are basically served by that web server. ''' try: if upload_path.startswith('ssh://'): # parse ssh://user@host/usr/local/autotest/packages hostline, remote_path = parse_ssh_path(upload_path) try: utils.run('scp %s %s:%s' % (file_path, hostline, remote_path)) r_path = os.path.join(remote_path, os.path.basename(file_path)) utils.run("ssh %s 'chmod 644 %s'" % (hostline, r_path)) except error.CmdError: logging.error("Error uploading to repository %s", upload_path) else: shutil.copy(file_path, upload_path) os.chmod(os.path.join(upload_path, os.path.basename(file_path)), 0644) except (IOError, os.error), why: logging.error("Upload of %s to %s failed: %s", file_path, upload_path, why) def upload_pkg_dir(self, dir_path, upload_path): ''' Upload a full directory. Depending on the upload path, the appropriate method for that protocol is called. Currently this copies the whole tmp package directory to the target directory. This assumes that the web server is running on the same machine where the method is being called from. The upload_path's files are basically served by that web server. ''' local_path = os.path.join(dir_path, "*") try: if upload_path.startswith('ssh://'): hostline, remote_path = parse_ssh_path(upload_path) try: utils.run('scp %s %s:%s' % (local_path, hostline, remote_path)) ssh_path = os.path.join(remote_path, "*") utils.run("ssh %s 'chmod 644 %s'" % (hostline, ssh_path)) except error.CmdError: logging.error("Error uploading to repository: %s", upload_path) else: utils.run("cp %s %s " % (local_path, upload_path)) up_path = os.path.join(upload_path, "*") utils.run("chmod 644 %s" % up_path) except (IOError, os.error), why: raise error.PackageUploadError("Upload of %s to %s failed: %s" % (dir_path, upload_path, why)) def remove_pkg(self, pkg_name, remove_path=None, remove_checksum=False): ''' Remove the package from the specified remove_path pkg_name : name of the package (ex: test-sleeptest.tar.bz2, dep-gcc.tar.bz2) remove_path : the location to remove the package from. ''' if remove_path: remove_path_list = [remove_path] elif len(self.upload_paths) > 0: remove_path_list = self.upload_paths else: raise error.PackageRemoveError( "Invalid path to remove the pkg from") checksum_path = self._get_checksum_file_path() if remove_checksum: self.remove_checksum(pkg_name) # remove the package and upload the checksum file to the repos for path in remove_path_list: self.remove_pkg_file(pkg_name, path) self.upload_pkg_file(checksum_path, path) def remove_pkg_file(self, filename, pkg_dir): ''' Remove the file named filename from pkg_dir ''' try: # Remove the file if pkg_dir.startswith('ssh://'): hostline, remote_path = parse_ssh_path(pkg_dir) path = os.path.join(remote_path, filename) utils.run("ssh %s 'rm -rf %s/%s'" % (hostline, remote_path, path)) else: os.remove(os.path.join(pkg_dir, filename)) except (IOError, os.error), why: raise error.PackageRemoveError("Could not remove %s from %s: %s " % (filename, pkg_dir, why)) def get_mirror_list(self, repo_urls): ''' Stub function for site specific mirrors. Returns: Priority ordered list ''' return repo_urls def _get_checksum_file_path(self): ''' Return the complete path of the checksum file (assumed to be stored in self.pkgmgr_dir ''' return os.path.join(self.pkgmgr_dir, CHECKSUM_FILE) def _get_checksum_dict(self): ''' Fetch the checksum file if not already fetched. If the checksum file cannot be fetched from the repos then a new file is created with the current package's (specified in pkg_path) checksum value in it. Populate the local checksum dictionary with the values read from the checksum file. The checksum file is assumed to be present in self.pkgmgr_dir ''' checksum_path = self._get_checksum_file_path() if not self._checksum_dict: # Fetch the checksum file try: try: self._run_command("ls %s" % checksum_path) except (error.CmdError, error.AutoservRunError): # The packages checksum file does not exist locally. # See if it is present in the repositories. self.fetch_pkg(CHECKSUM_FILE, checksum_path) except error.PackageFetchError: # This should not happen whilst fetching a package..if a # package is present in the repository, the corresponding # checksum file should also be automatically present. This # case happens only when a package # is being uploaded and if it is the first package to be # uploaded to the repos (hence no checksum file created yet) # Return an empty dictionary in that case return {} # Read the checksum file into memory checksum_file_contents = self._run_command('cat ' + checksum_path).stdout # Return {} if we have an empty checksum file present if not checksum_file_contents.strip(): return {} # Parse the checksum file contents into self._checksum_dict for line in checksum_file_contents.splitlines(): checksum, package_name = line.split(None, 1) self._checksum_dict[package_name] = checksum return self._checksum_dict def _save_checksum_dict(self, checksum_dict): ''' Save the checksum dictionary onto the checksum file. Update the local _checksum_dict variable with this new set of values. checksum_dict : New checksum dictionary checksum_dir : The directory in which to store the checksum file to. ''' checksum_path = self._get_checksum_file_path() self._checksum_dict = checksum_dict.copy() checksum_contents = '\n'.join(checksum + ' ' + pkg_name for pkg_name, checksum in checksum_dict.iteritems()) # Write the checksum file back to disk self._run_command('echo "%s" > %s' % (checksum_contents, checksum_path), _run_command_dargs={'verbose': False}) def compute_checksum(self, pkg_path): ''' Compute the MD5 checksum for the package file and return it. pkg_path : The complete path for the package file ''' md5sum_output = self._run_command("md5sum %s " % pkg_path).stdout return md5sum_output.split()[0] def update_checksum(self, pkg_path): ''' Update the checksum of the package in the packages' checksum file. This method is called whenever a package is fetched just to be sure that the checksums in the local file are the latest. pkg_path : The complete path to the package file. ''' # Compute the new checksum new_checksum = self.compute_checksum(pkg_path) checksum_dict = self._get_checksum_dict() checksum_dict[os.path.basename(pkg_path)] = new_checksum self._save_checksum_dict(checksum_dict) def remove_checksum(self, pkg_name): ''' Remove the checksum of the package from the packages checksum file. This method is called whenever a package is removed from the repositories in order clean its corresponding checksum. pkg_name : The name of the package to be removed ''' checksum_dict = self._get_checksum_dict() if pkg_name in checksum_dict: del checksum_dict[pkg_name] self._save_checksum_dict(checksum_dict) def compare_checksum(self, pkg_path): ''' Calculate the checksum of the file specified in pkg_path and compare it with the checksum in the checksum file Return True if both match else return False. pkg_path : The full path to the package file for which the checksum is being compared ''' checksum_dict = self._get_checksum_dict() package_name = os.path.basename(pkg_path) if not checksum_dict or package_name not in checksum_dict: return False repository_checksum = checksum_dict[package_name] local_checksum = self.compute_checksum(pkg_path) return (local_checksum == repository_checksum) def tar_package(self, pkg_name, src_dir, dest_dir, exclude_string=None): ''' Create a tar.bz2 file with the name 'pkg_name' say test-blah.tar.bz2. Excludes the directories specified in exclude_string while tarring the source. Returns the tarball path. ''' tarball_path = os.path.join(dest_dir, pkg_name) temp_path = tarball_path + '.tmp' cmd_list = ['tar', '-cf', temp_path, '-C', src_dir] if _PBZIP2_AVAILABLE: cmd_list.append('--use-compress-prog=pbzip2') else: cmd_list.append('-j') if exclude_string is not None: cmd_list.append(exclude_string) try: utils.system(' '.join(cmd_list)) except: os.unlink(temp_path) raise os.rename(temp_path, tarball_path) return tarball_path def untar_required(self, tarball_path, dest_dir): ''' Compare the checksum of the tarball_path with the .checksum file in the dest_dir and return False if it matches. The untar of the package happens only if the checksums do not match. ''' checksum_path = os.path.join(dest_dir, '.checksum') try: existing_checksum = self._run_command('cat ' + checksum_path).stdout except (error.CmdError, error.AutoservRunError): # If the .checksum file is not present (generally, this should # not be the case) then return True so that the untar happens return True new_checksum = self.compute_checksum(tarball_path) return (new_checksum.strip() != existing_checksum.strip()) def untar_pkg(self, tarball_path, dest_dir): ''' Untar the package present in the tarball_path and put a ".checksum" file in the dest_dir containing the checksum of the tarball. This method assumes that the package to be untarred is of the form .tar.bz2 ''' self._run_command('tar --no-same-owner -xjf %s -C %s' % (tarball_path, dest_dir)) # Put the .checksum file in the install_dir to note # where the package came from pkg_checksum = self.compute_checksum(tarball_path) pkg_checksum_path = os.path.join(dest_dir, '.checksum') self._run_command('echo "%s" > %s ' % (pkg_checksum, pkg_checksum_path)) @staticmethod def get_tarball_name(name, pkg_type): """Converts a package name and type into a tarball name. @param name: The name of the package @param pkg_type: The type of the package @returns A tarball filename for that specific type of package """ assert '-' not in pkg_type return '%s-%s.tar.bz2' % (pkg_type, name) @staticmethod def parse_tarball_name(tarball_name): """Coverts a package tarball name into a package name and type. @param tarball_name: The filename of the tarball @returns (name, pkg_type) where name is the package name and pkg_type is the package type. """ match = re.search(r'^([^-]*)-(.*)\.tar\.bz2$', tarball_name) pkg_type, name = match.groups() return name, pkg_type def is_url(self, url): """Return true if path looks like a URL""" return url.startswith('http://') def get_package_name(self, url, pkg_type): ''' Extract the group and test name for the url. This method is currently used only for tests. ''' if pkg_type == 'test': regex = '[^:]+://(.*)/([^/]*)$' return self._get_package_name(url, regex) else: return ('', url) def _get_package_name(self, url, regex): if not self.is_url(url): if url.endswith('.tar.bz2'): testname = url.replace('.tar.bz2', '') testname = re.sub(r'(\d*)\.', '', testname) return (testname, testname) else: return ('', url) match = re.match(regex, url) if not match: return ('', url) group, filename = match.groups() # Generate the group prefix. group = re.sub(r'\W', '_', group) # Drop the extension to get the raw test name. testname = re.sub(r'\.tar\.bz2', '', filename) # Drop any random numbers at the end of the test name if any testname = re.sub(r'\.(\d*)', '', testname) return (group, testname)