1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.config; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.config.OptionSetter.OptionFieldsForName; 21 import com.android.tradefed.config.remote.IRemoteFileResolver; 22 import com.android.tradefed.device.ITestDevice; 23 import com.android.tradefed.invoker.logger.CurrentInvocation; 24 import com.android.tradefed.invoker.logger.CurrentInvocation.InvocationInfo; 25 import com.android.tradefed.log.LogUtil.CLog; 26 import com.android.tradefed.result.error.InfraErrorIdentifier; 27 import com.android.tradefed.util.FileUtil; 28 import com.android.tradefed.util.MultiMap; 29 import com.android.tradefed.util.ZipUtil; 30 import com.android.tradefed.util.ZipUtil2; 31 32 import com.google.common.collect.ImmutableMap; 33 34 import java.io.File; 35 import java.io.IOException; 36 import java.lang.reflect.Field; 37 import java.net.URI; 38 import java.net.URISyntaxException; 39 import java.util.ArrayList; 40 import java.util.Collection; 41 import java.util.HashMap; 42 import java.util.HashSet; 43 import java.util.LinkedHashMap; 44 import java.util.List; 45 import java.util.Map; 46 import java.util.Map.Entry; 47 import java.util.ServiceLoader; 48 import java.util.Set; 49 import java.util.function.Supplier; 50 51 import javax.annotation.Nullable; 52 import javax.annotation.concurrent.GuardedBy; 53 import javax.annotation.concurrent.ThreadSafe; 54 55 /** 56 * Class that helps resolving path to remote files. 57 * 58 * <p>For example: gs://bucket/path/file.txt will be resolved by downloading the file from the GCS 59 * bucket. 60 * 61 * <p>New protocols should be added to META_INF/services. 62 */ 63 public class DynamicRemoteFileResolver { 64 65 // Query key for requesting to unzip a downloaded file automatically. 66 public static final String UNZIP_KEY = "unzip"; 67 // Query key for requesting a download to be optional, so if it fails we don't replace it. 68 public static final String OPTIONAL_KEY = "optional"; 69 70 private static final FileResolverLoader DEFAULT_FILE_RESOLVER_LOADER = 71 new ServiceFileResolverLoader(); 72 73 private final FileResolverLoader mFileResolverLoader; 74 75 private Map<String, OptionFieldsForName> mOptionMap; 76 // Populated from {@link ICommandOptions#getDynamicDownloadArgs()} 77 private Map<String, String> mExtraArgs = new LinkedHashMap<>(); 78 private ITestDevice mDevice; 79 DynamicRemoteFileResolver()80 public DynamicRemoteFileResolver() { 81 this(DEFAULT_FILE_RESOLVER_LOADER); 82 } 83 84 @VisibleForTesting DynamicRemoteFileResolver(FileResolverLoader loader)85 public DynamicRemoteFileResolver(FileResolverLoader loader) { 86 this.mFileResolverLoader = loader; 87 } 88 89 /** Sets the map of options coming from {@link OptionSetter} */ setOptionMap(Map<String, OptionFieldsForName> optionMap)90 public void setOptionMap(Map<String, OptionFieldsForName> optionMap) { 91 mOptionMap = optionMap; 92 } 93 94 /** Sets the device under tests */ setDevice(ITestDevice device)95 public void setDevice(ITestDevice device) { 96 mDevice = device; 97 } 98 99 /** Add extra args for the query. */ addExtraArgs(Map<String, String> extraArgs)100 public void addExtraArgs(Map<String, String> extraArgs) { 101 mExtraArgs.putAll(extraArgs); 102 } 103 104 /** 105 * Runs through all the {@link File} option type and check if their path should be resolved. 106 * 107 * @return The list of {@link File} that was resolved that way. 108 * @throws BuildRetrievalError 109 */ validateRemoteFilePath()110 public final Set<File> validateRemoteFilePath() throws BuildRetrievalError { 111 Set<File> downloadedFiles = new HashSet<>(); 112 try { 113 Map<Field, Object> fieldSeen = new HashMap<>(); 114 for (Map.Entry<String, OptionFieldsForName> optionPair : mOptionMap.entrySet()) { 115 final OptionFieldsForName optionFields = optionPair.getValue(); 116 for (Map.Entry<Object, Field> fieldEntry : optionFields) { 117 118 final Object obj = fieldEntry.getKey(); 119 final Field field = fieldEntry.getValue(); 120 final Option option = field.getAnnotation(Option.class); 121 if (option == null) { 122 continue; 123 } 124 // At this point, we know this is an option field; make sure it's set 125 field.setAccessible(true); 126 final Object value; 127 try { 128 value = field.get(obj); 129 if (value == null) { 130 continue; 131 } 132 } catch (IllegalAccessException e) { 133 throw new BuildRetrievalError( 134 String.format("internal error: %s", e.getMessage()), 135 InfraErrorIdentifier.ARTIFACT_UNSUPPORTED_PATH); 136 } 137 138 if (fieldSeen.get(field) != null && fieldSeen.get(field).equals(obj)) { 139 continue; 140 } 141 // Keep track of the field set on each object 142 fieldSeen.put(field, obj); 143 144 // The below contains unchecked casts that are mostly safe because we add/remove 145 // items of a type already in the collection; assuming they're not instances of 146 // some subclass of File. This is unlikely since we populate the items during 147 // option injection. The possibility still exists that constructors of 148 // initialized objects add objects that are instances of a File subclass. A 149 // safer approach would be to have a custom type that can be deferenced to 150 // access the resolved target file. This would also have the benefit of not 151 // having to modify any user collections and preserve the ordering. 152 153 if (value instanceof File) { 154 File consideredFile = (File) value; 155 File downloadedFile = resolveRemoteFiles(consideredFile, option); 156 if (downloadedFile != null) { 157 downloadedFiles.add(downloadedFile); 158 // Replace the field value 159 try { 160 field.set(obj, downloadedFile); 161 } catch (IllegalAccessException e) { 162 CLog.e(e); 163 throw new BuildRetrievalError( 164 String.format( 165 "Failed to download %s due to '%s'", 166 consideredFile.getPath(), e.getMessage()), 167 e); 168 } 169 } 170 } else if (value instanceof Collection) { 171 @SuppressWarnings("unchecked") // Mostly-safe, see above comment. 172 Collection<Object> c = (Collection<Object>) value; 173 Collection<Object> copy = new ArrayList<>(c); 174 for (Object o : copy) { 175 if (o instanceof File) { 176 File consideredFile = (File) o; 177 File downloadedFile = resolveRemoteFiles(consideredFile, option); 178 if (downloadedFile != null) { 179 downloadedFiles.add(downloadedFile); 180 // TODO: See if order could be preserved. 181 c.remove(consideredFile); 182 c.add(downloadedFile); 183 } 184 } 185 } 186 } else if (value instanceof Map) { 187 @SuppressWarnings("unchecked") // Mostly-safe, see above comment. 188 Map<Object, Object> m = (Map<Object, Object>) value; 189 Map<Object, Object> copy = new LinkedHashMap<>(m); 190 for (Entry<Object, Object> entry : copy.entrySet()) { 191 Object key = entry.getKey(); 192 Object val = entry.getValue(); 193 194 Object finalKey = key; 195 Object finalVal = val; 196 if (key instanceof File) { 197 key = resolveRemoteFiles((File) key, option); 198 if (key != null) { 199 downloadedFiles.add((File) key); 200 finalKey = key; 201 } 202 } 203 if (val instanceof File) { 204 val = resolveRemoteFiles((File) val, option); 205 if (val != null) { 206 downloadedFiles.add((File) val); 207 finalVal = val; 208 } 209 } 210 211 m.remove(entry.getKey()); 212 m.put(finalKey, finalVal); 213 } 214 } else if (value instanceof MultiMap) { 215 @SuppressWarnings("unchecked") // Mostly-safe, see above comment. 216 MultiMap<Object, Object> m = (MultiMap<Object, Object>) value; 217 synchronized (m) { 218 MultiMap<Object, Object> copy = new MultiMap<>(m); 219 for (Object key : copy.keySet()) { 220 List<Object> mapValues = copy.get(key); 221 222 m.remove(key); 223 Object finalKey = key; 224 if (key instanceof File) { 225 key = resolveRemoteFiles((File) key, option); 226 if (key != null) { 227 downloadedFiles.add((File) key); 228 finalKey = key; 229 } 230 } 231 for (Object mapValue : mapValues) { 232 if (mapValue instanceof File) { 233 File f = resolveRemoteFiles((File) mapValue, option); 234 if (f != null) { 235 downloadedFiles.add(f); 236 mapValue = f; 237 } 238 } 239 m.put(finalKey, mapValue); 240 } 241 } 242 } 243 } 244 } 245 } 246 } catch (RuntimeException | BuildRetrievalError e) { 247 // Clean up the files before throwing 248 for (File f : downloadedFiles) { 249 FileUtil.recursiveDelete(f); 250 } 251 throw e; 252 } 253 return downloadedFiles; 254 } 255 256 /** 257 * Download the files matching given filters in a remote zip file. 258 * 259 * <p>A file inside the remote zip file is only downloaded if its path matches any of the 260 * include filters but not the exclude filters. 261 * 262 * @param destDir the file to place the downloaded contents into. 263 * @param remoteZipFilePath the remote path to the zip file to download, relative to an 264 * implementation specific root. 265 * @param includeFilters a list of regex strings to download matching files. A file's path 266 * matching any filter will be downloaded. 267 * @param excludeFilters a list of regex strings to skip downloading matching files. A file's 268 * path matching any filter will not be downloaded. 269 * @throws BuildRetrievalError if files could not be downloaded. 270 */ resolvePartialDownloadZip( File destDir, String remoteZipFilePath, List<String> includeFilters, List<String> excludeFilters)271 public void resolvePartialDownloadZip( 272 File destDir, 273 String remoteZipFilePath, 274 List<String> includeFilters, 275 List<String> excludeFilters) 276 throws BuildRetrievalError { 277 Map<String, String> queryArgs; 278 String protocol; 279 try { 280 URI uri = new URI(remoteZipFilePath); 281 protocol = uri.getScheme(); 282 queryArgs = parseQuery(uri.getQuery()); 283 } catch (URISyntaxException e) { 284 throw new BuildRetrievalError( 285 String.format( 286 "Failed to parse the remote zip file path: %s", remoteZipFilePath), 287 e); 288 } 289 IRemoteFileResolver resolver = getResolver(protocol); 290 291 queryArgs.put("partial_download_dir", destDir.getAbsolutePath()); 292 if (includeFilters != null) { 293 queryArgs.put("include_filters", String.join(";", includeFilters)); 294 } 295 if (excludeFilters != null) { 296 queryArgs.put("exclude_filters", String.join(";", excludeFilters)); 297 } 298 // Downloaded individual files should be saved to destDir, return value is not needed. 299 try { 300 resolver.setPrimaryDevice(mDevice); 301 resolver.resolveRemoteFiles(new File(remoteZipFilePath), queryArgs); 302 } catch (BuildRetrievalError e) { 303 if (isOptional(queryArgs)) { 304 CLog.d( 305 "Failed to partially download '%s' but marked optional so skipping: %s", 306 remoteZipFilePath, e.getMessage()); 307 } else { 308 throw e; 309 } 310 } 311 } 312 getResolver(String protocol)313 protected IRemoteFileResolver getResolver(String protocol) { 314 return mFileResolverLoader.load(protocol, mExtraArgs); 315 } 316 317 @VisibleForTesting getGlobalConfig()318 IGlobalConfiguration getGlobalConfig() { 319 return GlobalConfiguration.getInstance(); 320 } 321 322 /** 323 * Utility that allows to check whether or not a file should be unzip and unzip it if required. 324 */ unzipIfRequired(File downloadedFile, Map<String, String> query)325 public static final File unzipIfRequired(File downloadedFile, Map<String, String> query) 326 throws IOException { 327 String unzipValue = query.get(UNZIP_KEY); 328 if (unzipValue != null && "true".equals(unzipValue.toLowerCase())) { 329 // File was requested to be unzipped. 330 if (ZipUtil.isZipFileValid(downloadedFile, false)) { 331 File extractedDir = 332 FileUtil.createTempDir( 333 FileUtil.getBaseName(downloadedFile.getName()), 334 CurrentInvocation.getInfo(InvocationInfo.WORK_FOLDER)); 335 ZipUtil2.extractZip(downloadedFile, extractedDir); 336 FileUtil.deleteFile(downloadedFile); 337 return extractedDir; 338 } else { 339 CLog.w("%s was requested to be unzipped but is not a valid zip.", downloadedFile); 340 } 341 } 342 // Return the original file untouched 343 return downloadedFile; 344 } 345 resolveRemoteFiles(File consideredFile, Option option)346 private File resolveRemoteFiles(File consideredFile, Option option) throws BuildRetrievalError { 347 File fileToResolve; 348 String path = consideredFile.getPath(); 349 String protocol; 350 Map<String, String> query; 351 try { 352 URI uri = new URI(path); 353 protocol = uri.getScheme(); 354 query = parseQuery(uri.getQuery()); 355 fileToResolve = new File(protocol + ":" + uri.getPath()); 356 } catch (URISyntaxException e) { 357 CLog.e(e); 358 return null; 359 } 360 IRemoteFileResolver resolver = getResolver(protocol); 361 if (resolver != null) { 362 try { 363 CLog.d( 364 "Considering option '%s' with path: '%s' for download.", 365 option.name(), path); 366 // Overrides query args 367 query.putAll(mExtraArgs); 368 resolver.setPrimaryDevice(mDevice); 369 return resolver.resolveRemoteFiles(fileToResolve, query); 370 } catch (BuildRetrievalError e) { 371 if (isOptional(query)) { 372 CLog.d( 373 "Failed to resolve '%s' but marked optional so skipping: %s", 374 fileToResolve, e.getMessage()); 375 } else { 376 throw e; 377 } 378 } 379 } 380 // Not a remote file 381 return null; 382 } 383 384 /** 385 * Parse a URL query style. Delimited by &, and map values represented by =. Example: 386 * ?key=value&key2=value2 387 */ parseQuery(String query)388 private Map<String, String> parseQuery(String query) { 389 Map<String, String> values = new HashMap<>(); 390 if (query == null) { 391 return values; 392 } 393 for (String maps : query.split("&")) { 394 String[] keyVal = maps.split("="); 395 values.put(keyVal[0], keyVal[1]); 396 } 397 return values; 398 } 399 400 /** Whether or not a link was requested as optional. */ isOptional(Map<String, String> query)401 private boolean isOptional(Map<String, String> query) { 402 String value = query.get(OPTIONAL_KEY); 403 if (value == null) { 404 return false; 405 } 406 return "true".equals(value.toLowerCase()); 407 } 408 409 /** Loads implementations of {@link IRemoteFileResolver}. */ 410 @VisibleForTesting 411 public interface FileResolverLoader { 412 /** 413 * Loads a resolver that can handle the provided scheme. 414 * 415 * @param scheme the URI scheme that the loaded resolver is expected to handle. 416 * @param config a map of all dynamic resolver configuration key-value pairs specified by 417 * the 'dynamic-resolver-args' TF command-line flag. 418 */ load(String scheme, Map<String, String> config)419 IRemoteFileResolver load(String scheme, Map<String, String> config); 420 } 421 422 /** 423 * Loads and caches file resolvers using the service loading facility. 424 * 425 * <p>This implementation uses the service loading facility to find and cache available 426 * resolvers on the first call to {@code load}. 427 * 428 * <p>This implementation is thread-safe and ensures that any loaded resolvers are loaded at 429 * most once per instance unless an exception is thrown. 430 */ 431 @ThreadSafe 432 @VisibleForTesting 433 static final class ServiceFileResolverLoader implements FileResolverLoader { 434 // We need the indirection since in production we use the context class loader that is 435 // defined when loading and not the one at construction. 436 private final Supplier<ClassLoader> mClassLoaderSupplier; 437 438 @GuardedBy("this") 439 private @Nullable ImmutableMap<String, IRemoteFileResolver> mResolvers; 440 ServiceFileResolverLoader()441 ServiceFileResolverLoader() { 442 mClassLoaderSupplier = () -> Thread.currentThread().getContextClassLoader(); 443 } 444 ServiceFileResolverLoader(ClassLoader classLoader)445 ServiceFileResolverLoader(ClassLoader classLoader) { 446 mClassLoaderSupplier = () -> classLoader; 447 } 448 449 @Override load(String scheme, Map<String, String> config)450 public synchronized IRemoteFileResolver load(String scheme, Map<String, String> config) { 451 if (mResolvers != null) { 452 return mResolvers.get(scheme); 453 } 454 455 // We use an intermediate map because the ImmutableMap builder throws if we add multiple 456 // entries with the same key. 457 Map<String, IRemoteFileResolver> resolvers = new HashMap<>(); 458 ServiceLoader<IRemoteFileResolver> serviceLoader = 459 ServiceLoader.load(IRemoteFileResolver.class, mClassLoaderSupplier.get()); 460 461 for (IRemoteFileResolver resolver : serviceLoader) { 462 resolvers.putIfAbsent(resolver.getSupportedProtocol(), resolver); 463 } 464 465 mResolvers = ImmutableMap.copyOf(resolvers); 466 return resolvers.get(scheme); 467 } 468 } 469 } 470