1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.util; 18 19 import com.android.tradefed.log.LogUtil.CLog; 20 21 import java.io.File; 22 import java.io.IOException; 23 import java.nio.file.Path; 24 import java.nio.file.Paths; 25 import java.util.ArrayList; 26 import java.util.Arrays; 27 import java.util.List; 28 import java.util.concurrent.TimeUnit; 29 import java.util.regex.Matcher; 30 import java.util.regex.Pattern; 31 32 /** 33 * File manager to download and upload files from Google Cloud Storage (GCS). 34 * 35 * <p>This class should NOT be used from the scope of a test (i.e., IRemoteTest). This is 36 * deprecated, please use {@link GCSFileDownloader} instead. 37 */ 38 @Deprecated 39 public class GCSBucketUtil { 40 41 // https://cloud.google.com/storage/docs/gsutil 42 43 private static final String CMD_COPY = "cp"; 44 private static final String CMD_MAKE_BUCKET = "mb"; 45 private static final String CMD_LS = "ls"; 46 private static final String CMD_STAT = "stat"; 47 private static final String CMD_HASH = "hash"; 48 private static final String CMD_REMOVE = "rm"; 49 private static final String CMD_REMOVE_BUCKET = "rb"; 50 private static final String CMD_VERSION = "-v"; 51 private static final String ENV_BOTO_PATH = "BOTO_PATH"; 52 private static final String ENV_BOTO_CONFIG = "BOTO_CONFIG"; 53 private static final String FILENAME_STDOUT = "-"; 54 private static final String FLAG_FORCE = "-f"; 55 private static final String FLAG_NO_CLOBBER = "-n"; 56 private static final String FLAG_PARALLEL = "-m"; 57 private static final String FLAG_PROJECT_ID = "-p"; 58 private static final String FLAG_RECURSIVE = "-r"; 59 private static final String GCS_SCHEME = "gs"; 60 private static final String GSUTIL = "gsutil"; 61 62 /** 63 * Whether gsutil is verified to be installed 64 */ 65 private static boolean mCheckedGsutil = false; 66 67 /** 68 * Number of attempts for gsutil operations. 69 * 70 * @see RunUtil#runTimedCmdRetry 71 */ 72 private int mAttempts = 1; 73 74 /** 75 * Path to the .boto files to use, set via environment variable $BOTO_PATH. 76 * 77 * @see <a href="https://cloud.google.com/storage/docs/gsutil/commands/config"> 78 * gsutil documentation</a> 79 */ 80 private String mBotoPath = null; 81 82 /** 83 * Path to the .boto file to use, set via environment variable $BOTO_CONFIG. 84 * 85 * @see <a href="https://cloud.google.com/storage/docs/gsutil/commands/config"> 86 * gsutil documentation</a> 87 */ 88 private String mBotoConfig = null; 89 90 /** 91 * Name of the GCS bucket. 92 */ 93 private String mBucketName = null; 94 95 /** 96 * Whether to use the "-n" flag to avoid clobbering files. 97 */ 98 private boolean mNoClobber = false; 99 100 /** 101 * Whether to use the "-m" flag to parallelize large operations. 102 */ 103 private boolean mParallel = false; 104 105 /** 106 * Whether to use the "-r" flag to perform a recursive copy. 107 */ 108 private boolean mRecursive = true; 109 110 /** 111 * Retry interval for gsutil operations. 112 * 113 * @see RunUtil#runTimedCmdRetry 114 */ 115 private long mRetryInterval = 0; 116 117 /** 118 * Timeout for gsutil operations. 119 * 120 * @see RunUtil#runTimedCmdRetry 121 */ 122 private long mTimeoutMs = 0; 123 GCSBucketUtil(String bucketName)124 public GCSBucketUtil(String bucketName) { 125 setBucketName(bucketName); 126 } 127 128 /** 129 * Verify that gsutil is installed. 130 */ checkGSUtil()131 void checkGSUtil() throws IOException { 132 if (mCheckedGsutil) { 133 return; 134 } 135 136 // N.B. We don't use retry / attempts here, since this doesn't involve any RPC. 137 CommandResult res = getRunUtil() 138 .runTimedCmd(mTimeoutMs, GSUTIL, CMD_VERSION); 139 140 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 141 throw new IOException( 142 "gsutil is not installed.\n" 143 + "https://cloud.google.com/storage/docs/gsutil for instructions."); 144 } 145 146 mCheckedGsutil = true; 147 } 148 149 /** 150 * Copy a file or directory to or from the bucket. 151 * 152 * @param source Source file or pattern 153 * @param dest Destination file or pattern 154 * @return {@link CommandResult} result of the operation. 155 */ copy(String source, String dest)156 public CommandResult copy(String source, String dest) throws IOException { 157 checkGSUtil(); 158 CLog.d("Copying %s => %s", source, dest); 159 160 IRunUtil run = getRunUtil(); 161 List<String> command = new ArrayList<>(); 162 163 command.add(GSUTIL); 164 165 if (mParallel) { 166 command.add(FLAG_PARALLEL); 167 } 168 169 command.add(CMD_COPY); 170 171 if (mRecursive) { 172 command.add(FLAG_RECURSIVE); 173 } 174 175 if (mNoClobber) { 176 command.add(FLAG_NO_CLOBBER); 177 } 178 179 command.add(source); 180 command.add(dest); 181 182 String[] commandAsStr = command.toArray(new String[0]); 183 184 CommandResult res = run 185 .runTimedCmdRetry(mTimeoutMs, mRetryInterval, mAttempts, commandAsStr); 186 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 187 throw new IOException( 188 String.format( 189 "Failed to copy '%s' -> '%s' with %s\nstdout: %s\nstderr: %s", 190 source, 191 dest, 192 res.getStatus(), 193 res.getStdout(), 194 res.getStderr())); 195 } 196 return res; 197 } 198 getAttempts()199 public int getAttempts() { 200 return mAttempts; 201 } 202 getBotoConfig()203 public String getBotoConfig() { 204 return mBotoConfig; 205 } 206 getBotoPath()207 public String getBotoPath() { 208 return mBotoPath; 209 } 210 getBucketName()211 public String getBucketName() { 212 return mBucketName; 213 } 214 getNoClobber()215 public boolean getNoClobber() { 216 return mNoClobber; 217 } 218 getParallel()219 public boolean getParallel() { 220 return mParallel; 221 } 222 getRecursive()223 public boolean getRecursive() { 224 return mRecursive; 225 } 226 getRetryInterval()227 public long getRetryInterval() { 228 return mRetryInterval; 229 } 230 getRunUtil()231 protected IRunUtil getRunUtil() { 232 IRunUtil run = new RunUtil(); 233 234 if (mBotoPath != null) { 235 run.setEnvVariable(ENV_BOTO_PATH, mBotoPath); 236 } 237 238 if (mBotoConfig != null) { 239 run.setEnvVariable(ENV_BOTO_CONFIG, mBotoConfig); 240 } 241 242 return run; 243 } 244 getTimeout()245 public long getTimeout() { 246 return mTimeoutMs; 247 } 248 249 /** 250 * Retrieve the gs://bucket/path URI 251 */ getUriForGcsPath(Path path)252 String getUriForGcsPath(Path path) { 253 // N.B. Would just use java.net.URI, but it doesn't allow e.g. underscores, 254 // which are valid in GCS bucket names. 255 if (!path.isAbsolute()) { 256 path = Paths.get("/").resolve(path); 257 } 258 return String.format("%s://%s%s", GCS_SCHEME, mBucketName, path.toString()); 259 } 260 261 /** 262 * Make the GCS bucket. 263 * 264 * @return {@link CommandResult} result of the operation. 265 * @throws IOException 266 */ makeBucket(String projectId)267 public CommandResult makeBucket(String projectId) throws IOException { 268 checkGSUtil(); 269 CLog.d("Making bucket %s for project %s", mBucketName, projectId); 270 271 List<String> command = new ArrayList<>(); 272 command.add(GSUTIL); 273 command.add(CMD_MAKE_BUCKET); 274 275 if (projectId != null) { 276 command.add(FLAG_PROJECT_ID); 277 command.add(projectId); 278 } 279 280 command.add(getUriForGcsPath(Paths.get("/"))); 281 282 CommandResult res = getRunUtil() 283 .runTimedCmdRetry(mTimeoutMs, mRetryInterval, mAttempts, 284 command.toArray(new String[0])); 285 286 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 287 throw new IOException( 288 String.format( 289 "Failed to create bucket '%s' with %s\nstdout: %s\nstderr: %s", 290 mBucketName, 291 res.getStatus(), 292 res.getStdout(), 293 res.getStderr())); 294 } 295 296 return res; 297 } 298 299 /** 300 * List files under a GCS path. 301 * 302 * @param bucketPath the GCS path 303 * @return a list of {@link String}s that are files under the GCS path 304 * @throws IOException 305 */ ls(Path bucketPath)306 public List<String> ls(Path bucketPath) throws IOException { 307 checkGSUtil(); 308 CLog.d("Check stat of %s %s", mBucketName, bucketPath); 309 310 List<String> command = new ArrayList<>(); 311 command.add(GSUTIL); 312 command.add(CMD_LS); 313 314 command.add(getUriForGcsPath(bucketPath)); 315 316 CommandResult res = 317 getRunUtil() 318 .runTimedCmdRetry( 319 mTimeoutMs, 320 mRetryInterval, 321 mAttempts, 322 command.toArray(new String[0])); 323 324 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 325 throw new IOException( 326 String.format( 327 "Failed to list path '%s %s' with %s\nstdout: %s\nstderr: %s", 328 mBucketName, 329 bucketPath, 330 res.getStatus(), 331 res.getStdout(), 332 res.getStderr())); 333 } 334 return Arrays.asList(res.getStdout().split("\n")); 335 } 336 337 /** 338 * Check a GCS file is a file or not a file (a folder). 339 * 340 * <p>If the filename ends with '/', then it's a folder. gsutil ls gs://filename should return 341 * the gs://filename if it's a file. gsutil ls gs://folder name should return the files in the 342 * folder if there are files in the folder. And it will return gs://folder/ if there is no files 343 * in the folder. 344 * 345 * @param path the path relative to bucket.. 346 * @return it's a file or not a file. 347 * @throws IOException 348 */ isFile(String path)349 public boolean isFile(String path) throws IOException { 350 if (path.endsWith("/")) { 351 return false; 352 } 353 List<String> files = ls(Paths.get(path)); 354 if (files.size() > 1) { 355 return false; 356 } 357 if (files.size() == 1) { 358 return files.get(0).equals(getUriForGcsPath(Paths.get(path))); 359 } 360 return false; 361 } 362 363 /** Simple wrapper for file info in GCS. */ 364 public static class GCSFileMetadata { 365 public String mName; 366 public String mMd5Hash = null; 367 GCSFileMetadata()368 private GCSFileMetadata() {} 369 370 /** 371 * Parse a string to a {@link GCSFileMetadata} object. 372 * 373 * @param statOutput 374 * @return {@link GCSFileMetadata} 375 */ parseStat(String statOutput)376 public static GCSFileMetadata parseStat(String statOutput) { 377 GCSFileMetadata info = new GCSFileMetadata(); 378 String[] infoLines = statOutput.split("\n"); 379 // Remove the trail ':' 380 info.mName = infoLines[0].substring(0, infoLines[0].length() - 1); 381 for (String line : infoLines) { 382 String[] keyValue = line.split(":", 2); 383 String key = keyValue[0].trim(); 384 String value = keyValue[1].trim(); 385 386 if ("Hash (md5)".equals(key)) { 387 info.mMd5Hash = value; 388 } 389 } 390 return info; 391 } 392 } 393 394 /** 395 * Get the state of the file for the GCS path. 396 * 397 * @param bucketPath the GCS path 398 * @return {@link GCSFileMetadata} for the GCS path 399 * @throws IOException 400 */ stat(Path bucketPath)401 public GCSFileMetadata stat(Path bucketPath) throws IOException { 402 checkGSUtil(); 403 CLog.d("Check stat of %s %s", mBucketName, bucketPath); 404 405 List<String> command = new ArrayList<>(); 406 command.add(GSUTIL); 407 command.add(CMD_STAT); 408 409 command.add(getUriForGcsPath(bucketPath)); 410 411 // The stat output will be something like: 412 // gs://bucketName/file.txt: 413 // Creation time: Tue, 14 Aug 2018 00:20:48 GMT 414 // Update time: Tue, 14 Aug 2018 16:58:39 GMT 415 // Storage class: STANDARD 416 // Content-Length: 1097 417 // Content-Type: text/x-sh 418 // Hash (crc32c): WutM7Q== 419 // Hash (md5): GZX0xHUXtGnoKIGTDk6Pbg== 420 // ETag: CKKNu/Si69wCEAU= 421 // Generation: 1534206048913058 422 // Metageneration: 5 423 CommandResult res = 424 getRunUtil() 425 .runTimedCmdRetry( 426 mTimeoutMs, 427 mRetryInterval, 428 mAttempts, 429 command.toArray(new String[0])); 430 431 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 432 throw new IOException( 433 String.format( 434 "Failed to stat path '%s %s' with %s\nstdout: %s\nstderr: %s", 435 mBucketName, 436 bucketPath, 437 res.getStatus(), 438 res.getStdout(), 439 res.getStderr())); 440 } 441 return GCSFileMetadata.parseStat(res.getStdout()); 442 } 443 444 /** 445 * Calculate the md5 hash for the local file. 446 * 447 * @param localFile a local file 448 * @return the md5 hash for the local file. 449 * @throws IOException 450 */ md5Hash(File localFile)451 public String md5Hash(File localFile) throws IOException { 452 checkGSUtil(); 453 List<String> command = new ArrayList<>(); 454 command.add(GSUTIL); 455 command.add(CMD_HASH); 456 command.add("-m"); 457 command.add(localFile.getAbsolutePath()); 458 459 CommandResult res = 460 getRunUtil() 461 .runTimedCmdRetry( 462 mTimeoutMs, 463 mRetryInterval, 464 mAttempts, 465 command.toArray(new String[0])); 466 467 if (CommandStatus.SUCCESS.equals(res.getStatus())) { 468 // An example output of "gustil hash -m file": 469 // Hashes [base64] for error_prone_rules.mk: 470 // Hash (md5): eHfvTtNyH/x3GcyfApEIDQ== 471 // 472 // Operation completed over 1 objects/2.0 KiB. 473 Pattern md5Pattern = 474 Pattern.compile( 475 ".*Hash\\s*\\(md5\\)\\:\\s*(.*?)\n.*", 476 Pattern.MULTILINE | Pattern.DOTALL); 477 Matcher matcher = md5Pattern.matcher(res.getStdout()); 478 if (matcher.find()) { 479 return matcher.group(1); 480 } 481 } 482 throw new IOException( 483 String.format( 484 "Failed to calculate md5 hash for '%s' with %s\nstdout: %s\nstderr: %s", 485 localFile.getAbsoluteFile(), 486 res.getStatus(), 487 res.getStdout(), 488 res.getStderr())); 489 } 490 491 /** 492 * Download a file or directory from a GCS bucket to the current directory. 493 * 494 * @param bucketPath File path in the GCS bucket 495 * @return {@link CommandResult} result of the operation. 496 */ pull(Path bucketPath)497 public CommandResult pull(Path bucketPath) throws IOException { 498 return copy(getUriForGcsPath(bucketPath), "."); 499 } 500 501 /** 502 * Download a file or directory from a GCS bucket. 503 * 504 * @param bucketPath File path in the GCS bucket 505 * @param localFile Local destination path 506 * @return {@link CommandResult} result of the operation. 507 */ pull(Path bucketPath, File localFile)508 public CommandResult pull(Path bucketPath, File localFile) throws IOException { 509 return copy(getUriForGcsPath(bucketPath), localFile.getPath()); 510 } 511 512 /** 513 * Download a file from a GCS bucket, and extract its contents. 514 * 515 * @param bucketPath File path in the GCS bucket 516 * @return String contents of the file 517 */ pullContents(Path bucketPath)518 public String pullContents(Path bucketPath) throws IOException { 519 CommandResult res = copy(getUriForGcsPath(bucketPath), FILENAME_STDOUT); 520 return res.getStdout(); 521 } 522 523 /** 524 * Upload a local file or directory to a GCS bucket. 525 * 526 * @param localFile Local file or directory 527 * @return {@link CommandResult} result of the operation. 528 */ push(File localFile)529 public CommandResult push(File localFile) throws IOException { 530 return push(localFile, Paths.get("/")); 531 } 532 533 /** 534 * Upload a local file or directory to a GCS bucket with a specific path. 535 * 536 * @param localFile Local file or directory 537 * @param bucketPath File path in the GCS bucket 538 * @return {@link CommandResult} result of the operation. 539 */ push(File localFile, Path bucketPath)540 public CommandResult push(File localFile, Path bucketPath) throws IOException { 541 return copy(localFile.getAbsolutePath(), getUriForGcsPath(bucketPath)); 542 } 543 544 /** 545 * Upload a String to a GCS bucket. 546 * 547 * @param contents File contents, as a string 548 * @param bucketPath File path in the GCS bucket 549 * @return {@link CommandResult} result of the operation. 550 */ pushString(String contents, Path bucketPath)551 public CommandResult pushString(String contents, Path bucketPath) throws IOException { 552 File localFile = null; 553 try { 554 localFile = FileUtil.createTempFile(mBucketName, null); 555 FileUtil.writeToFile(contents, localFile); 556 return copy(localFile.getAbsolutePath(), getUriForGcsPath(bucketPath)); 557 } finally { 558 FileUtil.deleteFile(localFile); 559 } 560 } 561 562 /** 563 * Remove a file or directory from the bucket. 564 * 565 * @param pattern File, directory, or pattern to remove. 566 * @param force Whether to ignore failures and continue silently (will not throw) 567 */ remove(String pattern, boolean force)568 public CommandResult remove(String pattern, boolean force) throws IOException { 569 checkGSUtil(); 570 String path = getUriForGcsPath(Paths.get(pattern)); 571 CLog.d("Removing file(s) %s", path); 572 573 List<String> command = new ArrayList<>(); 574 command.add(GSUTIL); 575 command.add(CMD_REMOVE); 576 577 if (mRecursive) { 578 command.add(FLAG_RECURSIVE); 579 } 580 581 if (force) { 582 command.add(FLAG_FORCE); 583 } 584 585 command.add(path); 586 587 CommandResult res = getRunUtil() 588 .runTimedCmdRetry(mTimeoutMs, mRetryInterval, mAttempts, 589 command.toArray(new String[0])); 590 591 if (!force && !CommandStatus.SUCCESS.equals(res.getStatus())) { 592 throw new IOException( 593 String.format( 594 "Failed to remove '%s' with %s\nstdout: %s\nstderr: %s", 595 pattern, 596 res.getStatus(), 597 res.getStdout(), 598 res.getStderr())); 599 } 600 return res; 601 } 602 603 /** 604 * Remove a file or directory from the bucket. 605 * 606 * @param pattern File, directory, or pattern to remove. 607 */ remove(String pattern)608 public CommandResult remove(String pattern) throws IOException { 609 return remove(pattern, false); 610 } 611 612 /** 613 * Remove a file or directory from the bucket. 614 * 615 * @param path Path to remove 616 * @param force Whether to fail if the file does not exist 617 */ remove(Path path, boolean force)618 public CommandResult remove(Path path, boolean force) throws IOException { 619 return remove(path.toString(), force); 620 } 621 622 /** 623 * Remove a file or directory from the bucket. 624 * 625 * @param path Path to remove 626 */ remove(Path path)627 public CommandResult remove(Path path) throws IOException { 628 return remove(path.toString(), false); 629 } 630 631 632 /** 633 * Remove the GCS bucket 634 * 635 * @throws IOException 636 */ removeBucket()637 public CommandResult removeBucket() throws IOException { 638 checkGSUtil(); 639 CLog.d("Removing bucket %s", mBucketName); 640 641 String[] command = { 642 GSUTIL, 643 CMD_REMOVE_BUCKET, 644 getUriForGcsPath(Paths.get("/")) 645 }; 646 647 CommandResult res = getRunUtil() 648 .runTimedCmdRetry(mTimeoutMs, mRetryInterval, mAttempts, command); 649 650 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 651 throw new IOException( 652 String.format( 653 "Failed to remove bucket '%s' with %s\nstdout: %s\nstderr: %s", 654 mBucketName, 655 res.getStatus(), 656 res.getStdout(), 657 res.getStderr())); 658 } 659 660 return res; 661 } 662 setAttempts(int attempts)663 public void setAttempts(int attempts) { 664 mAttempts = attempts; 665 } 666 setBotoConfig(String botoConfig)667 public void setBotoConfig(String botoConfig) { 668 mBotoConfig = botoConfig; 669 } 670 setBotoPath(String botoPath)671 public void setBotoPath(String botoPath) { 672 mBotoPath = botoPath; 673 } 674 setBucketName(String bucketName)675 public void setBucketName(String bucketName) { 676 mBucketName = bucketName; 677 } 678 setNoClobber(boolean noClobber)679 public void setNoClobber(boolean noClobber) { 680 mNoClobber = noClobber; 681 } 682 setParallel(boolean parallel)683 public void setParallel(boolean parallel) { 684 mParallel = parallel; 685 } 686 setRecursive(boolean recursive)687 public void setRecursive(boolean recursive) { 688 mRecursive = recursive; 689 } 690 setRetryInterval(long retryInterval)691 public void setRetryInterval(long retryInterval) { 692 mRetryInterval = retryInterval; 693 } 694 setTimeoutMs(long timeout)695 public void setTimeoutMs(long timeout) { 696 mTimeoutMs = timeout; 697 } 698 setTimeout(long timeout, TimeUnit unit)699 public void setTimeout(long timeout, TimeUnit unit) { 700 setTimeoutMs(unit.toMillis(timeout)); 701 } 702 } 703