1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.device.cloud; 17 18 import com.android.tradefed.build.IBuildInfo; 19 import com.android.tradefed.command.remote.DeviceDescriptor; 20 import com.android.tradefed.device.TestDeviceOptions; 21 import com.android.tradefed.device.cloud.AcloudConfigParser.AcloudKeys; 22 import com.android.tradefed.device.cloud.GceAvdInfo.GceStatus; 23 import com.android.tradefed.log.ITestLogger; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.result.ByteArrayInputStreamSource; 26 import com.android.tradefed.result.FileInputStreamSource; 27 import com.android.tradefed.result.InputStreamSource; 28 import com.android.tradefed.result.LogDataType; 29 import com.android.tradefed.targetprep.TargetSetupError; 30 import com.android.tradefed.util.ArrayUtil; 31 import com.android.tradefed.util.CommandResult; 32 import com.android.tradefed.util.CommandStatus; 33 import com.android.tradefed.util.FileUtil; 34 import com.android.tradefed.util.GoogleApiClientUtil; 35 import com.android.tradefed.util.IRunUtil; 36 import com.android.tradefed.util.RunUtil; 37 38 import com.google.api.client.auth.oauth2.Credential; 39 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; 40 import com.google.api.client.json.JsonFactory; 41 import com.google.api.client.json.jackson2.JacksonFactory; 42 import com.google.api.services.compute.Compute; 43 import com.google.api.services.compute.Compute.Instances.GetSerialPortOutput; 44 import com.google.api.services.compute.ComputeScopes; 45 import com.google.api.services.compute.model.SerialPortOutput; 46 import com.google.common.annotations.VisibleForTesting; 47 import com.google.common.net.HostAndPort; 48 49 import java.io.File; 50 import java.io.IOException; 51 import java.lang.ProcessBuilder.Redirect; 52 import java.security.GeneralSecurityException; 53 import java.time.Duration; 54 import java.util.Arrays; 55 import java.util.List; 56 import java.util.regex.Matcher; 57 import java.util.regex.Pattern; 58 59 /** Helper that manages the GCE calls to start/stop and collect logs from GCE. */ 60 public class GceManager { 61 public static final String GCE_INSTANCE_NAME_KEY = "gce-instance-name"; 62 public static final String GCE_INSTANCE_CLEANED_KEY = "gce-instance-clean-called"; 63 64 private static final long BUGREPORT_TIMEOUT = 15 * 60 * 1000L; 65 private static final long REMOTE_FILE_OP_TIMEOUT = 10 * 60 * 1000L; 66 private static final Pattern BUGREPORTZ_RESPONSE_PATTERN = Pattern.compile("(OK:)(.*)"); 67 private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); 68 private static final List<String> SCOPES = Arrays.asList(ComputeScopes.COMPUTE_READONLY); 69 70 private DeviceDescriptor mDeviceDescriptor; 71 private TestDeviceOptions mDeviceOptions; 72 private IBuildInfo mBuildInfo; 73 74 private String mGceInstanceName = null; 75 private String mGceHost = null; 76 private GceAvdInfo mGceAvdInfo = null; 77 78 /** 79 * Ctor 80 * 81 * @param deviceDesc The {@link DeviceDescriptor} that will be associated with the GCE device. 82 * @param deviceOptions A {@link TestDeviceOptions} associated with the device. 83 * @param buildInfo A {@link IBuildInfo} describing the gce build to start. 84 */ GceManager( DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo)85 public GceManager( 86 DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo) { 87 mDeviceDescriptor = deviceDesc; 88 mDeviceOptions = deviceOptions; 89 mBuildInfo = buildInfo; 90 91 if (!deviceOptions.allowGceCmdTimeoutOverride()) { 92 return; 93 } 94 int index = deviceOptions.getGceDriverParams().lastIndexOf("--boot-timeout"); 95 if (index != -1 && deviceOptions.getGceDriverParams().size() > index + 1) { 96 String driverTimeoutStringSec = deviceOptions.getGceDriverParams().get(index + 1); 97 try { 98 // Add some extra time on top of Acloud: acloud boot the device then we expect 99 // the Tradefed online check to take a bit of time, use 3min as a safe overhead 100 long driverTimeoutMs = 101 Long.parseLong(driverTimeoutStringSec) * 1000 + 3 * 60 * 1000; 102 long gceCmdTimeoutMs = deviceOptions.getGceCmdTimeout(); 103 deviceOptions.setGceCmdTimeout(driverTimeoutMs); 104 CLog.i( 105 "Replacing --gce-boot-timeout %s by --boot-timeout %s.", 106 gceCmdTimeoutMs, driverTimeoutMs); 107 } catch (NumberFormatException e) { 108 CLog.e(e); 109 } 110 } 111 } 112 113 /** @deprecated Use other constructors, we keep this temporarily for backward compatibility. */ 114 @Deprecated GceManager( DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo, List<IBuildInfo> testResourceBuildInfos)115 public GceManager( 116 DeviceDescriptor deviceDesc, 117 TestDeviceOptions deviceOptions, 118 IBuildInfo buildInfo, 119 List<IBuildInfo> testResourceBuildInfos) { 120 this(deviceDesc, deviceOptions, buildInfo); 121 } 122 123 /** 124 * Ctor, variation that can be used to provide the GCE instance name to use directly. 125 * 126 * @param deviceDesc The {@link DeviceDescriptor} that will be associated with the GCE device. 127 * @param deviceOptions A {@link TestDeviceOptions} associated with the device 128 * @param buildInfo A {@link IBuildInfo} describing the gce build to start. 129 * @param gceInstanceName The instance name to use. 130 * @param gceHost The host name or ip of the instance to use. 131 */ GceManager( DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo, String gceInstanceName, String gceHost)132 public GceManager( 133 DeviceDescriptor deviceDesc, 134 TestDeviceOptions deviceOptions, 135 IBuildInfo buildInfo, 136 String gceInstanceName, 137 String gceHost) { 138 this(deviceDesc, deviceOptions, buildInfo); 139 mGceInstanceName = gceInstanceName; 140 mGceHost = gceHost; 141 } 142 startGce()143 public GceAvdInfo startGce() throws TargetSetupError { 144 return startGce(null); 145 } 146 147 /** 148 * Attempt to start a gce instance 149 * 150 * @return a {@link GceAvdInfo} describing the GCE instance. Could be a BOOT_FAIL instance. 151 * @throws TargetSetupError 152 */ startGce(String ipDevice)153 public GceAvdInfo startGce(String ipDevice) throws TargetSetupError { 154 mGceAvdInfo = null; 155 // For debugging purposes bypass. 156 if (mGceHost != null && mGceInstanceName != null) { 157 mGceAvdInfo = 158 new GceAvdInfo( 159 mGceInstanceName, 160 HostAndPort.fromString(mGceHost) 161 .withDefaultPort(mDeviceOptions.getRemoteAdbPort())); 162 return mGceAvdInfo; 163 } 164 // Add extra args. 165 File reportFile = null; 166 try { 167 reportFile = FileUtil.createTempFile("gce_avd_driver", ".json"); 168 List<String> gceArgs = buildGceCmd(reportFile, mBuildInfo, ipDevice); 169 170 long driverTimeoutMs = getTestDeviceOptions().getGceCmdTimeout(); 171 if (!getTestDeviceOptions().allowGceCmdTimeoutOverride()) { 172 long driverTimeoutSec = 173 Duration.ofMillis(driverTimeoutMs - 3 * 60 * 1000).toSeconds(); 174 // --boot-timeout takes a value in seconds 175 gceArgs.add("--boot-timeout"); 176 gceArgs.add(Long.toString(driverTimeoutSec)); 177 driverTimeoutMs = driverTimeoutSec * 1000; 178 } 179 180 CLog.i("Launching GCE with %s", gceArgs.toString()); 181 CommandResult cmd = 182 getRunUtil() 183 .runTimedCmd( 184 getTestDeviceOptions().getGceCmdTimeout(), 185 gceArgs.toArray(new String[gceArgs.size()])); 186 CLog.i("GCE driver stderr: %s", cmd.getStderr()); 187 String instanceName = extractInstanceName(cmd.getStderr()); 188 if (instanceName != null) { 189 mBuildInfo.addBuildAttribute(GCE_INSTANCE_NAME_KEY, instanceName); 190 } else { 191 CLog.w("Could not extract an instance name for the gce device."); 192 } 193 if (CommandStatus.TIMED_OUT.equals(cmd.getStatus())) { 194 String errors = 195 String.format( 196 "acloud errors: timeout after %dms, acloud did not return", 197 driverTimeoutMs); 198 if (instanceName != null) { 199 // If we managed to parse the instance name, report the boot failure so it 200 // can be shutdown. 201 mGceAvdInfo = new GceAvdInfo(instanceName, null, errors, GceStatus.BOOT_FAIL); 202 return mGceAvdInfo; 203 } 204 throw new TargetSetupError(errors, mDeviceDescriptor); 205 } else if (!CommandStatus.SUCCESS.equals(cmd.getStatus())) { 206 CLog.w("Error when booting the Gce instance, reading output of gce driver"); 207 mGceAvdInfo = 208 GceAvdInfo.parseGceInfoFromFile( 209 reportFile, mDeviceDescriptor, mDeviceOptions.getRemoteAdbPort()); 210 String errors = ""; 211 if (mGceAvdInfo != null) { 212 // We always return the GceAvdInfo describing the instance when possible 213 // The caller can decide actions to be taken. 214 return mGceAvdInfo; 215 } else { 216 errors = 217 "Could not get a valid instance name, check the gce driver's output." 218 + "The instance may not have booted up at all."; 219 CLog.e(errors); 220 throw new TargetSetupError( 221 String.format("acloud errors: %s", errors), mDeviceDescriptor); 222 } 223 } 224 mGceAvdInfo = 225 GceAvdInfo.parseGceInfoFromFile( 226 reportFile, mDeviceDescriptor, mDeviceOptions.getRemoteAdbPort()); 227 return mGceAvdInfo; 228 } catch (IOException e) { 229 throw new TargetSetupError("failed to create log file", e, mDeviceDescriptor); 230 } finally { 231 FileUtil.deleteFile(reportFile); 232 } 233 } 234 235 /** 236 * Retrieve the instance name from the gce boot logs. Search for the 'name': 'gce-<name>' 237 * pattern to extract the name of it. We extract from the logs instead of result file because on 238 * gce boot failure, the attempted instance name won't show in json. 239 */ extractInstanceName(String bootupLogs)240 protected String extractInstanceName(String bootupLogs) { 241 if (bootupLogs != null) { 242 final String pattern = "'name': u?'(((gce-)|(ins-))(.*?))'"; 243 Pattern namePattern = Pattern.compile(pattern); 244 Matcher matcher = namePattern.matcher(bootupLogs); 245 if (matcher.find()) { 246 return matcher.group(1); 247 } 248 } 249 return null; 250 } 251 252 /** Build and return the command to launch GCE. Exposed for testing. */ buildGceCmd(File reportFile, IBuildInfo b, String ipDevice)253 protected List<String> buildGceCmd(File reportFile, IBuildInfo b, String ipDevice) { 254 File avdDriverFile = getTestDeviceOptions().getAvdDriverBinary(); 255 if (!avdDriverFile.exists()) { 256 throw new RuntimeException( 257 String.format( 258 "Could not find the Acloud driver at %s", 259 avdDriverFile.getAbsolutePath())); 260 } 261 if (!avdDriverFile.canExecute()) { 262 // Set the executable bit if needed 263 FileUtil.chmodGroupRWX(avdDriverFile); 264 } 265 List<String> gceArgs = ArrayUtil.list(avdDriverFile.getAbsolutePath()); 266 gceArgs.add( 267 TestDeviceOptions.getCreateCommandByInstanceType( 268 getTestDeviceOptions().getInstanceType())); 269 // Handle the build id related params 270 List<String> gceDriverParams = getTestDeviceOptions().getGceDriverParams(); 271 272 if (TestDeviceOptions.InstanceType.CHEEPS.equals( 273 getTestDeviceOptions().getInstanceType())) { 274 gceArgs.add("--avd-type"); 275 gceArgs.add("cheeps"); 276 277 if (getTestDeviceOptions().getCrosUser() != null 278 && getTestDeviceOptions().getCrosPassword() != null) { 279 gceArgs.add("--user"); 280 gceArgs.add(getTestDeviceOptions().getCrosUser()); 281 gceArgs.add("--password"); 282 gceArgs.add(getTestDeviceOptions().getCrosPassword()); 283 } 284 } 285 286 // If args passed by gce-driver-param do not contain build_id or branch, 287 // use build_id and branch from device BuildInfo 288 if (!gceDriverParams.contains("--build_id") && !gceDriverParams.contains("--branch")) { 289 gceArgs.add("--build_target"); 290 if (b.getBuildAttributes().containsKey("build_target")) { 291 // If BuildInfo contains the attribute for a build target, use that. 292 gceArgs.add(b.getBuildAttributes().get("build_target")); 293 } else { 294 gceArgs.add(b.getBuildFlavor()); 295 } 296 gceArgs.add("--branch"); 297 gceArgs.add(b.getBuildBranch()); 298 gceArgs.add("--build_id"); 299 gceArgs.add(b.getBuildId()); 300 } 301 // Add additional args passed by gce-driver-param. 302 gceArgs.addAll(gceDriverParams); 303 // Get extra params by instance type 304 gceArgs.addAll( 305 TestDeviceOptions.getExtraParamsByInstanceType( 306 getTestDeviceOptions().getInstanceType(), 307 getTestDeviceOptions().getBaseImage())); 308 if (ipDevice == null) { 309 gceArgs.add("--config_file"); 310 gceArgs.add(getAvdConfigFile().getAbsolutePath()); 311 if (getTestDeviceOptions().getServiceAccountJsonKeyFile() != null) { 312 gceArgs.add("--service_account_json_private_key_path"); 313 gceArgs.add( 314 getTestDeviceOptions().getServiceAccountJsonKeyFile().getAbsolutePath()); 315 } 316 } else { 317 gceArgs.add("--host"); 318 gceArgs.add(ipDevice); 319 } 320 gceArgs.add("--report_file"); 321 gceArgs.add(reportFile.getAbsolutePath()); 322 switch (getTestDeviceOptions().getGceDriverLogLevel()) { 323 case DEBUG: 324 gceArgs.add("-v"); 325 break; 326 case VERBOSE: 327 gceArgs.add("-vv"); 328 break; 329 default: 330 break; 331 } 332 if (getTestDeviceOptions().getGceAccount() != null) { 333 gceArgs.add("--email"); 334 gceArgs.add(getTestDeviceOptions().getGceAccount()); 335 } 336 // Do not pass flags --logcat_file and --serial_log_file to collect logcat and serial logs. 337 338 return gceArgs; 339 } 340 341 /** 342 * Shutdown the Gce instance associated with the {@link #startGce()}. 343 * 344 * @return returns true if gce shutdown was requested as non-blocking. 345 */ shutdownGce()346 public boolean shutdownGce() { 347 if (!getTestDeviceOptions().getAvdDriverBinary().canExecute()) { 348 mGceAvdInfo = null; 349 throw new RuntimeException( 350 String.format( 351 "GCE launcher %s is invalid", 352 getTestDeviceOptions().getAvdDriverBinary())); 353 } 354 String instanceName = null; 355 boolean notFromGceAvd = false; 356 if (mGceAvdInfo != null) { 357 instanceName = mGceAvdInfo.instanceName(); 358 } 359 if (instanceName == null) { 360 instanceName = mBuildInfo.getBuildAttributes().get(GCE_INSTANCE_NAME_KEY); 361 notFromGceAvd = true; 362 } 363 if (instanceName == null) { 364 CLog.d("No instance to shutdown."); 365 return false; 366 } 367 try { 368 boolean res = AcloudShutdown(getTestDeviceOptions(), getRunUtil(), instanceName); 369 // Be more lenient if instance name was not reported officially and we still attempt 370 // to clean it. 371 if (res || notFromGceAvd) { 372 mBuildInfo.addBuildAttribute(GCE_INSTANCE_CLEANED_KEY, "true"); 373 } 374 return res; 375 } finally { 376 mGceAvdInfo = null; 377 } 378 } 379 380 /** 381 * Actual Acloud run to shutdown the virtual device. 382 * 383 * @param options The {@link TestDeviceOptions} for the Acloud options 384 * @param runUtil The {@link IRunUtil} to run Acloud 385 * @param instanceName The instance to shutdown. 386 * @return True if successful 387 */ AcloudShutdown( TestDeviceOptions options, IRunUtil runUtil, String instanceName)388 public static boolean AcloudShutdown( 389 TestDeviceOptions options, IRunUtil runUtil, String instanceName) { 390 List<String> gceArgs = ArrayUtil.list(options.getAvdDriverBinary().getAbsolutePath()); 391 gceArgs.add("delete"); 392 // Add extra args. 393 File f = null; 394 File config = null; 395 try { 396 config = FileUtil.createTempFile(options.getAvdConfigFile().getName(), "config"); 397 gceArgs.add("--instance_names"); 398 gceArgs.add(instanceName); 399 gceArgs.add("--config_file"); 400 // Copy the config in case it comes from a dynamic file. In order to ensure Acloud has 401 // the file until it's done with it. 402 FileUtil.copyFile(options.getAvdConfigFile(), config); 403 gceArgs.add(config.getAbsolutePath()); 404 if (options.getServiceAccountJsonKeyFile() != null) { 405 gceArgs.add("--service_account_json_private_key_path"); 406 gceArgs.add(options.getServiceAccountJsonKeyFile().getAbsolutePath()); 407 } 408 f = FileUtil.createTempFile("gce_avd_driver", ".json"); 409 gceArgs.add("--report_file"); 410 gceArgs.add(f.getAbsolutePath()); 411 CLog.i("Tear down of GCE with %s", gceArgs.toString()); 412 if (options.waitForGceTearDown()) { 413 CommandResult cmd = 414 runUtil.runTimedCmd( 415 options.getGceCmdTimeout(), 416 gceArgs.toArray(new String[gceArgs.size()])); 417 FileUtil.deleteFile(config); 418 if (!CommandStatus.SUCCESS.equals(cmd.getStatus())) { 419 CLog.w( 420 "Failed to tear down GCE %s with the following arg: %s." 421 + "\nstdout:%s\nstderr:%s", 422 instanceName, gceArgs, cmd.getStdout(), cmd.getStderr()); 423 return false; 424 } 425 } else { 426 // Discard the output so the process is not linked to the parent and doesn't die 427 // if the JVM exit. 428 Process p = runUtil.runCmdInBackground(Redirect.DISCARD, gceArgs); 429 AcloudDeleteCleaner cleaner = new AcloudDeleteCleaner(p, config); 430 cleaner.start(); 431 } 432 } catch (IOException | RuntimeException e) { 433 CLog.e("failed to create log file for GCE Teardown"); 434 CLog.e(e); 435 FileUtil.deleteFile(config); 436 return false; 437 } finally { 438 FileUtil.deleteFile(f); 439 } 440 return true; 441 } 442 443 /** 444 * Get a bugreportz from the device using ssh to avoid any adb connection potential issue. 445 * 446 * @param gceAvd The {@link GceAvdInfo} that describe the device. 447 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 448 * GCE device. 449 * @param runUtil a {@link IRunUtil} to execute commands. 450 * @return A file pointing to the zip bugreport, or null if an issue occurred. 451 * @throws IOException 452 */ getBugreportzWithSsh( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil)453 public static File getBugreportzWithSsh( 454 GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil) throws IOException { 455 String output = remoteSshCommandExec(gceAvd, options, runUtil, "bugreportz"); 456 Matcher match = BUGREPORTZ_RESPONSE_PATTERN.matcher(output); 457 if (!match.find()) { 458 CLog.e("Something went wrong during bugreportz collection: '%s'", output); 459 return null; 460 } 461 String remoteFilePath = match.group(2); 462 File localTmpFile = FileUtil.createTempFile("bugreport-ssh", ".zip"); 463 if (!RemoteFileUtil.fetchRemoteFile( 464 gceAvd, options, runUtil, REMOTE_FILE_OP_TIMEOUT, remoteFilePath, localTmpFile)) { 465 FileUtil.deleteFile(localTmpFile); 466 return null; 467 } 468 return localTmpFile; 469 } 470 471 /** 472 * Get a bugreport via ssh for a nested instance. This requires requesting the adb in the nested 473 * virtual instance. 474 * 475 * @param gceAvd The {@link GceAvdInfo} that describe the device. 476 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 477 * GCE device. 478 * @param runUtil a {@link IRunUtil} to execute commands. 479 * @return A file pointing to the zip bugreport, or null if an issue occurred. 480 * @throws IOException 481 */ getNestedDeviceSshBugreportz( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil)482 public static File getNestedDeviceSshBugreportz( 483 GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil) throws IOException { 484 if (gceAvd == null || gceAvd.hostAndPort() == null) { 485 return null; 486 } 487 String output = 488 remoteSshCommandExec( 489 gceAvd, 490 options, 491 runUtil, 492 "./bin/adb", 493 "wait-for-device", 494 "shell", 495 "bugreportz"); 496 Matcher match = BUGREPORTZ_RESPONSE_PATTERN.matcher(output); 497 if (!match.find()) { 498 CLog.e("Something went wrong during bugreportz collection: '%s'", output); 499 return null; 500 } 501 String deviceFilePath = match.group(2); 502 String pullOutput = 503 remoteSshCommandExec(gceAvd, options, runUtil, "./bin/adb", "pull", deviceFilePath); 504 CLog.d(pullOutput); 505 String remoteFilePath = "./" + new File(deviceFilePath).getName(); 506 File localTmpFile = FileUtil.createTempFile("bugreport-ssh", ".zip"); 507 if (!RemoteFileUtil.fetchRemoteFile( 508 gceAvd, options, runUtil, REMOTE_FILE_OP_TIMEOUT, remoteFilePath, localTmpFile)) { 509 FileUtil.deleteFile(localTmpFile); 510 return null; 511 } 512 return localTmpFile; 513 } 514 515 /** 516 * Fetch a remote file from a nested instance and log it. 517 * 518 * @param logger The {@link ITestLogger} where to log the file. 519 * @param gceAvd The {@link GceAvdInfo} that describe the device. 520 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 521 * GCE device. 522 * @param runUtil a {@link IRunUtil} to execute commands. 523 * @param remoteFilePath The remote path where to find the file. 524 * @param type the {@link LogDataType} of the logged file. 525 */ logNestedRemoteFile( ITestLogger logger, GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String remoteFilePath, LogDataType type)526 public static void logNestedRemoteFile( 527 ITestLogger logger, 528 GceAvdInfo gceAvd, 529 TestDeviceOptions options, 530 IRunUtil runUtil, 531 String remoteFilePath, 532 LogDataType type) { 533 logNestedRemoteFile(logger, gceAvd, options, runUtil, remoteFilePath, type, null); 534 } 535 536 /** 537 * Fetch a remote file from a nested instance and log it. 538 * 539 * @param logger The {@link ITestLogger} where to log the file. 540 * @param gceAvd The {@link GceAvdInfo} that describe the device. 541 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 542 * GCE device. 543 * @param runUtil a {@link IRunUtil} to execute commands. 544 * @param remoteFilePath The remote path where to find the file. 545 * @param type the {@link LogDataType} of the logged file. 546 * @param baseName The base name to use to log the file. If null the actual file name will be 547 * used. 548 */ logNestedRemoteFile( ITestLogger logger, GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String remoteFilePath, LogDataType type, String baseName)549 public static void logNestedRemoteFile( 550 ITestLogger logger, 551 GceAvdInfo gceAvd, 552 TestDeviceOptions options, 553 IRunUtil runUtil, 554 String remoteFilePath, 555 LogDataType type, 556 String baseName) { 557 File remoteFile = 558 RemoteFileUtil.fetchRemoteFile( 559 gceAvd, options, runUtil, REMOTE_FILE_OP_TIMEOUT, remoteFilePath); 560 if (remoteFile != null) { 561 // If we happened to fetch a directory, log all the subfiles 562 logFile(remoteFile, baseName, logger, type); 563 } 564 } 565 logFile( File remoteFile, String baseName, ITestLogger logger, LogDataType type)566 private static void logFile( 567 File remoteFile, String baseName, ITestLogger logger, LogDataType type) { 568 if (remoteFile.isDirectory()) { 569 for (File f : remoteFile.listFiles()) { 570 logFile(f, null, logger, type); 571 } 572 } else { 573 try (InputStreamSource remoteFileStream = new FileInputStreamSource(remoteFile, true)) { 574 String name = baseName; 575 if (name == null) { 576 name = remoteFile.getName(); 577 } 578 logger.testLog(name, type, remoteFileStream); 579 } 580 } 581 } 582 583 /** 584 * Execute the remote command via ssh on an instance. 585 * 586 * @param gceAvd The {@link GceAvdInfo} that describe the device. 587 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 588 * GCE device. 589 * @param runUtil a {@link IRunUtil} to execute commands. 590 * @param timeoutMs The timeout in millisecond for the command. 0 means no timeout. 591 * @param command The remote command to execute. 592 * @return {@link CommandResult} containing the result of the execution. 593 */ remoteSshCommandExecution( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, long timeoutMs, String... command)594 public static CommandResult remoteSshCommandExecution( 595 GceAvdInfo gceAvd, 596 TestDeviceOptions options, 597 IRunUtil runUtil, 598 long timeoutMs, 599 String... command) { 600 return RemoteSshUtil.remoteSshCommandExec(gceAvd, options, runUtil, timeoutMs, command); 601 } 602 remoteSshCommandExec( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String... command)603 private static String remoteSshCommandExec( 604 GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String... command) { 605 CommandResult res = 606 remoteSshCommandExecution(gceAvd, options, runUtil, BUGREPORT_TIMEOUT, command); 607 // We attempt to get a clean output from our command 608 String output = res.getStdout().trim(); 609 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 610 CLog.e("issue when attempting to execute '%s':", Arrays.asList(command)); 611 CLog.e("Stderr: %s", res.getStderr()); 612 } else if (output.isEmpty()) { 613 CLog.e("Stdout from '%s' was empty", Arrays.asList(command)); 614 CLog.e("Stderr: %s", res.getStderr()); 615 } 616 return output; 617 } 618 619 /** 620 * Reads the current content of the Gce Avd instance serial log. 621 * 622 * @param infos The {@link GceAvdInfo} describing the instance. 623 * @param avdConfigFile the avd config file 624 * @param jsonKeyFile the service account json key file. 625 * @param runUtil a {@link IRunUtil} to execute commands. 626 * @return The serial log output or null if something goes wrong. 627 */ getInstanceSerialLog( GceAvdInfo infos, File avdConfigFile, File jsonKeyFile, IRunUtil runUtil)628 public static String getInstanceSerialLog( 629 GceAvdInfo infos, File avdConfigFile, File jsonKeyFile, IRunUtil runUtil) { 630 AcloudConfigParser config = AcloudConfigParser.parseConfig(avdConfigFile); 631 if (config == null) { 632 CLog.e("Failed to parse our acloud config."); 633 return null; 634 } 635 if (infos == null) { 636 return null; 637 } 638 try { 639 Credential credential = createCredential(config, jsonKeyFile); 640 String project = config.getValueForKey(AcloudKeys.PROJECT); 641 String zone = config.getValueForKey(AcloudKeys.ZONE); 642 String instanceName = infos.instanceName(); 643 Compute compute = 644 new Compute.Builder( 645 GoogleNetHttpTransport.newTrustedTransport(), 646 JSON_FACTORY, 647 null) 648 .setApplicationName(project) 649 .setHttpRequestInitializer(credential) 650 .build(); 651 GetSerialPortOutput outputPort = 652 compute.instances().getSerialPortOutput(project, zone, instanceName); 653 SerialPortOutput output = outputPort.execute(); 654 return output.getContents(); 655 } catch (GeneralSecurityException | IOException e) { 656 CLog.e(e); 657 return null; 658 } 659 } 660 createCredential(AcloudConfigParser config, File jsonKeyFile)661 private static Credential createCredential(AcloudConfigParser config, File jsonKeyFile) 662 throws GeneralSecurityException, IOException { 663 if (jsonKeyFile != null) { 664 return GoogleApiClientUtil.createCredentialFromJsonKeyFile(jsonKeyFile, SCOPES); 665 } else if (config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_JSON_PRIVATE_KEY) != null) { 666 jsonKeyFile = 667 new File(config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_JSON_PRIVATE_KEY)); 668 return GoogleApiClientUtil.createCredentialFromJsonKeyFile(jsonKeyFile, SCOPES); 669 } else { 670 String serviceAccount = config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_NAME); 671 String serviceKey = config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_PRIVATE_KEY); 672 return GoogleApiClientUtil.createCredentialFromP12File( 673 serviceAccount, new File(serviceKey), SCOPES); 674 } 675 } 676 cleanUp()677 public void cleanUp() { 678 // Clean up logs file if any was created. 679 } 680 681 /** Returns the instance of the {@link IRunUtil}. */ 682 @VisibleForTesting getRunUtil()683 IRunUtil getRunUtil() { 684 return RunUtil.getDefault(); 685 } 686 687 /** 688 * Log the serial output of a device described by {@link GceAvdInfo}. 689 * 690 * @param infos The {@link GceAvdInfo} describing the instance. 691 * @param logger The {@link ITestLogger} where to log the serial log. 692 */ logSerialOutput(GceAvdInfo infos, ITestLogger logger)693 public void logSerialOutput(GceAvdInfo infos, ITestLogger logger) { 694 String output = 695 GceManager.getInstanceSerialLog( 696 infos, 697 getAvdConfigFile(), 698 getTestDeviceOptions().getServiceAccountJsonKeyFile(), 699 getRunUtil()); 700 if (output == null) { 701 CLog.w("Failed to collect the instance serial logs."); 702 return; 703 } 704 try (ByteArrayInputStreamSource source = 705 new ByteArrayInputStreamSource(output.getBytes())) { 706 logger.testLog("gce_full_serial_log", LogDataType.TEXT, source); 707 } 708 } 709 710 /** Log the information related to the stable host image used. */ logStableHostImageInfos(IBuildInfo build)711 public void logStableHostImageInfos(IBuildInfo build) { 712 AcloudConfigParser config = AcloudConfigParser.parseConfig(getAvdConfigFile()); 713 if (config == null) { 714 CLog.e("Failed to parse our acloud config."); 715 return; 716 } 717 if (build == null) { 718 return; 719 } 720 if (config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_NAME) != null) { 721 build.addBuildAttribute( 722 AcloudKeys.STABLE_HOST_IMAGE_NAME.toString(), 723 config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_NAME)); 724 } 725 if (config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_PROJECT) != null) { 726 build.addBuildAttribute( 727 AcloudKeys.STABLE_HOST_IMAGE_PROJECT.toString(), 728 config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_PROJECT)); 729 } 730 } 731 732 /** 733 * Returns the {@link TestDeviceOptions} associated with the device that the gce manager was 734 * initialized with. 735 */ getTestDeviceOptions()736 private TestDeviceOptions getTestDeviceOptions() { 737 return mDeviceOptions; 738 } 739 740 @VisibleForTesting getAvdConfigFile()741 File getAvdConfigFile() { 742 return getTestDeviceOptions().getAvdConfigFile(); 743 } 744 745 /** 746 * Thread that helps cleaning the copied config when the process is done. This ensures acloud is 747 * not missing its config until its done. 748 */ 749 private static class AcloudDeleteCleaner extends Thread { 750 private Process mProcess; 751 private File mConfigFile; 752 AcloudDeleteCleaner(Process p, File config)753 public AcloudDeleteCleaner(Process p, File config) { 754 setDaemon(true); 755 setName("acloud-delete-cleaner"); 756 mProcess = p; 757 mConfigFile = config; 758 } 759 760 @Override run()761 public void run() { 762 try { 763 mProcess.waitFor(); 764 } catch (InterruptedException e) { 765 CLog.e(e); 766 } 767 FileUtil.deleteFile(mConfigFile); 768 } 769 } 770 } 771