1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.build.StubBuildProvider; 22 import com.android.tradefed.clearcut.ClearcutClient; 23 import com.android.tradefed.command.CommandOptions; 24 import com.android.tradefed.command.CommandRunner; 25 import com.android.tradefed.config.ConfigurationException; 26 import com.android.tradefed.config.GlobalConfiguration; 27 import com.android.tradefed.config.IConfiguration; 28 import com.android.tradefed.config.IDeviceConfiguration; 29 import com.android.tradefed.config.OptionCopier; 30 import com.android.tradefed.config.OptionSetter; 31 import com.android.tradefed.device.DeviceNotAvailableException; 32 import com.android.tradefed.device.DeviceSelectionOptions; 33 import com.android.tradefed.device.ITestDevice; 34 import com.android.tradefed.device.TestDeviceOptions; 35 import com.android.tradefed.device.cloud.GceAvdInfo; 36 import com.android.tradefed.device.cloud.GceManager; 37 import com.android.tradefed.device.cloud.ManagedRemoteDevice; 38 import com.android.tradefed.device.cloud.RemoteFileUtil; 39 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 40 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 41 import com.android.tradefed.log.ITestLogger; 42 import com.android.tradefed.log.LogUtil.CLog; 43 import com.android.tradefed.result.FailureDescription; 44 import com.android.tradefed.result.FileInputStreamSource; 45 import com.android.tradefed.result.ITestInvocationListener; 46 import com.android.tradefed.result.InputStreamSource; 47 import com.android.tradefed.result.LogDataType; 48 import com.android.tradefed.result.proto.FileProtoResultReporter; 49 import com.android.tradefed.result.proto.ProtoResultParser; 50 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus; 51 import com.android.tradefed.targetprep.BuildError; 52 import com.android.tradefed.targetprep.TargetSetupError; 53 import com.android.tradefed.testtype.SubprocessTfLauncher; 54 import com.android.tradefed.util.CommandResult; 55 import com.android.tradefed.util.CommandStatus; 56 import com.android.tradefed.util.FileUtil; 57 import com.android.tradefed.util.IRunUtil; 58 import com.android.tradefed.util.RunUtil; 59 import com.android.tradefed.util.SystemUtil; 60 import com.android.tradefed.util.TimeUtil; 61 import com.android.tradefed.util.proto.TestRecordProtoUtil; 62 63 import com.google.common.base.Strings; 64 import com.google.protobuf.InvalidProtocolBufferException; 65 66 import java.io.File; 67 import java.io.IOException; 68 import java.io.PrintWriter; 69 import java.util.ArrayList; 70 import java.util.Arrays; 71 import java.util.HashSet; 72 import java.util.List; 73 74 /** Implementation of {@link InvocationExecution} that drives a remote execution. */ 75 public class RemoteInvocationExecution extends InvocationExecution { 76 77 public static final long PUSH_TF_TIMEOUT = 150000L; 78 public static final long PULL_RESULT_TIMEOUT = 180000L; 79 public static final long REMOTE_PROCESS_RUNNING_WAIT = 15000L; 80 public static final long LAUNCH_EXTRA_DEVICE = 15 * 60 * 1000L; 81 public static final long SETUP_REMOTE_DIR_TIMEOUT = 10 * 60 * 1000L; 82 public static final long NEW_USER_TIMEOUT = 5 * 60 * 1000L; 83 public static final long JOIN_CLEAN_TIMEOUT_MS = 2 * 60 * 1000L; 84 85 public static final String REMOTE_USER_DIR = "/home/{$USER}/"; 86 public static final String PROTO_RESULT_NAME = "output.pb"; 87 public static final String STDOUT_FILE = "screen-VM_tradefed-stdout.txt"; 88 public static final String STDERR_FILE = "screen-VM_tradefed-stderr.txt"; 89 public static final String REMOTE_CONFIG = "configuration"; 90 public static final String GLOBAL_REMOTE_CONFIG = "global-remote-configuration"; 91 92 private static final int MAX_CONNECTION_REFUSED_COUNT = 3; 93 private static final int MAX_PUSH_TF_ATTEMPTS = 3; 94 private static final String TRADEFED_EARLY_TERMINATION = 95 "Remote Tradefed might have terminated early.\nRemote Stderr:\n%s"; 96 97 private String mRemoteTradefedDir = null; 98 private String mRemoteAdbPath = null; 99 private ProtoResultParser mProtoParser = null; 100 private String mRemoteConsoleStdErr = null; 101 102 @Override fetchBuild( TestInformation testInfo, IConfiguration config, IRescheduler rescheduler, ITestInvocationListener listener)103 public boolean fetchBuild( 104 TestInformation testInfo, 105 IConfiguration config, 106 IRescheduler rescheduler, 107 ITestInvocationListener listener) 108 throws DeviceNotAvailableException, BuildRetrievalError { 109 // TODO: handle multiple devices/build config 110 StubBuildProvider stubProvider = new StubBuildProvider(); 111 112 String deviceName = config.getDeviceConfig().get(0).getDeviceName(); 113 OptionCopier.copyOptionsNoThrow( 114 config.getDeviceConfig().get(0).getBuildProvider(), stubProvider); 115 116 IBuildInfo info = stubProvider.getBuild(); 117 if (info == null) { 118 return false; 119 } 120 testInfo.getContext().addDeviceBuildInfo(deviceName, info); 121 updateBuild(info, config); 122 return true; 123 } 124 125 @Override customizeDevicePreInvocation(IConfiguration config, IInvocationContext context)126 protected void customizeDevicePreInvocation(IConfiguration config, IInvocationContext context) { 127 super.customizeDevicePreInvocation(config, context); 128 129 if (config.getCommandOptions().getShardCount() != null 130 && config.getCommandOptions().getShardIndex() == null) { 131 ITestDevice device = context.getDevices().get(0); 132 TestDeviceOptions options = device.getOptions(); 133 // Trigger the multi-tenant start in the VM 134 options.addGceDriverParams("--num-avds-per-instance"); 135 String count = config.getCommandOptions().getShardCount().toString(); 136 options.addGceDriverParams(count); 137 InvocationMetricLogger.addInvocationMetrics( 138 InvocationMetricKey.CF_INSTANCE_COUNT, count); 139 } 140 } 141 142 @Override runTests( TestInformation info, IConfiguration config, ITestInvocationListener listener)143 public void runTests( 144 TestInformation info, IConfiguration config, ITestInvocationListener listener) 145 throws Throwable { 146 ManagedRemoteDevice device = (ManagedRemoteDevice) info.getDevice(); 147 GceAvdInfo gceInfo = device.getRemoteAvdInfo(); 148 149 // Run remote TF (new tests?) 150 IRunUtil runUtil = new RunUtil(); 151 152 TestDeviceOptions options = device.getOptions(); 153 String mainRemoteDir = getRemoteMainDir(options); 154 mRemoteAdbPath = String.format("/home/%s/bin/adb", options.getInstanceUser()); 155 // Select the TF version that should be pushed to the remote VM 156 File tfToPush = getLocalTradefedPath(listener, options.getRemoteTf()); 157 if (tfToPush == null) { 158 return; 159 } 160 161 mRemoteTradefedDir = mainRemoteDir + "tradefed/"; 162 CommandResult createRemoteDir = 163 GceManager.remoteSshCommandExecution( 164 gceInfo, options, runUtil, 120000L, "mkdir", "-p", mRemoteTradefedDir); 165 if (!CommandStatus.SUCCESS.equals(createRemoteDir.getStatus())) { 166 listener.invocationFailed( 167 createInvocationFailure( 168 "Failed to create remote dir.", FailureStatus.INFRA_FAILURE)); 169 return; 170 } 171 172 // Push Tradefed to the remote 173 int attempt = 0; 174 boolean result = false; 175 while (!result && attempt < MAX_PUSH_TF_ATTEMPTS) { 176 result = 177 RemoteFileUtil.pushFileToRemote( 178 gceInfo, 179 options, 180 Arrays.asList("-r"), 181 runUtil, 182 PUSH_TF_TIMEOUT, 183 mRemoteTradefedDir, 184 tfToPush); 185 attempt++; 186 } 187 if (!result) { 188 CLog.e("Failed to push Tradefed."); 189 listener.invocationFailed( 190 createInvocationFailure( 191 "Failed to push Tradefed.", FailureStatus.INFRA_FAILURE)); 192 return; 193 } 194 195 mRemoteTradefedDir = mRemoteTradefedDir + tfToPush.getName() + "/"; 196 CommandResult listRemoteDir = 197 GceManager.remoteSshCommandExecution( 198 gceInfo, options, runUtil, 120000L, "ls", "-l", mRemoteTradefedDir); 199 CLog.d("stdout: %s", listRemoteDir.getStdout()); 200 CLog.d("stderr: %s", listRemoteDir.getStderr()); 201 202 File configFile = createRemoteConfig(config, listener, mRemoteTradefedDir); 203 File globalConfig = null; 204 try { 205 CLog.d("Pushing Tradefed XML configuration to remote."); 206 boolean resultPush = 207 RemoteFileUtil.pushFileToRemote( 208 gceInfo, 209 options, 210 null, 211 runUtil, 212 PUSH_TF_TIMEOUT, 213 mRemoteTradefedDir, 214 configFile); 215 if (!resultPush) { 216 CLog.e("Failed to push Tradefed Configuration."); 217 listener.invocationFailed( 218 createInvocationFailure( 219 "Failed to push Tradefed Configuration.", 220 FailureStatus.INFRA_FAILURE)); 221 return; 222 } 223 224 String[] whitelistConfigs = 225 new String[] { 226 GlobalConfiguration.SANDBOX_FACTORY_TYPE_NAME, 227 GlobalConfiguration.HOST_OPTIONS_TYPE_NAME, 228 "android-build" 229 }; 230 try { 231 globalConfig = 232 GlobalConfiguration.getInstance() 233 .cloneConfigWithFilter(new HashSet<>(), whitelistConfigs); 234 } catch (IOException e) { 235 listener.invocationFailed(createInvocationFailure(e, FailureStatus.INFRA_FAILURE)); 236 return; 237 } 238 try (InputStreamSource source = new FileInputStreamSource(globalConfig)) { 239 listener.testLog(GLOBAL_REMOTE_CONFIG, LogDataType.XML, source); 240 } 241 // Push the global configuration 242 boolean resultPushGlobal = 243 RemoteFileUtil.pushFileToRemote( 244 gceInfo, 245 options, 246 null, 247 runUtil, 248 PUSH_TF_TIMEOUT, 249 mRemoteTradefedDir, 250 globalConfig); 251 if (!resultPushGlobal) { 252 CLog.e("Failed to push Tradefed Global Configuration."); 253 listener.invocationFailed( 254 createInvocationFailure( 255 "Failed to push Tradefed Global Configuration.", 256 FailureStatus.INFRA_FAILURE)); 257 return; 258 } 259 260 resetAdb(gceInfo, options, runUtil); 261 runRemote( 262 listener, 263 info.getContext(), 264 configFile, 265 gceInfo, 266 options, 267 runUtil, 268 config, 269 globalConfig); 270 collectAdbLogs(gceInfo, options, runUtil, listener); 271 } finally { 272 FileUtil.recursiveDelete(configFile); 273 FileUtil.recursiveDelete(globalConfig); 274 } 275 } 276 277 @Override doSetup(TestInformation testInfo, IConfiguration config, ITestLogger logger)278 public void doSetup(TestInformation testInfo, IConfiguration config, ITestLogger logger) 279 throws TargetSetupError, BuildError, DeviceNotAvailableException { 280 // Skip 281 } 282 283 @Override doTeardown( TestInformation testInfo, IConfiguration config, ITestLogger logger, Throwable exception)284 public void doTeardown( 285 TestInformation testInfo, 286 IConfiguration config, 287 ITestLogger logger, 288 Throwable exception) 289 throws Throwable { 290 super.runDevicePostInvocationTearDown(testInfo.getContext(), config, exception); 291 } 292 293 @Override doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception)294 public void doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception) { 295 // Skip 296 } 297 298 @Override getAdbVersion()299 protected String getAdbVersion() { 300 // Do not report the adb version from the parent, the remote child will remote its own. 301 return null; 302 } 303 runRemote( ITestInvocationListener currentInvocationListener, IInvocationContext context, File configFile, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, IConfiguration config, File globalConfig)304 private void runRemote( 305 ITestInvocationListener currentInvocationListener, 306 IInvocationContext context, 307 File configFile, 308 GceAvdInfo info, 309 TestDeviceOptions options, 310 IRunUtil runUtil, 311 IConfiguration config, 312 File globalConfig) 313 throws InvalidProtocolBufferException, IOException { 314 List<String> remoteTfCommand = new ArrayList<>(); 315 remoteTfCommand.add("pushd"); 316 remoteTfCommand.add(mRemoteTradefedDir + ";"); 317 remoteTfCommand.add(String.format("PATH=%s:$PATH", new File(mRemoteAdbPath).getParent())); 318 remoteTfCommand.add("screen -dmSU tradefed sh -c"); 319 320 StringBuilder tfCmdBuilder = 321 new StringBuilder("TF_GLOBAL_CONFIG=" + globalConfig.getName()); 322 // Set an env variable to notify that this a remote environment. 323 tfCmdBuilder.append(" " + SystemUtil.REMOTE_VM_VARIABLE + "=1"); 324 // Disable clearcut in the remote 325 tfCmdBuilder.append(" " + ClearcutClient.DISABLE_CLEARCUT_KEY + "=1"); 326 tfCmdBuilder.append(" ENTRY_CLASS=" + CommandRunner.class.getCanonicalName()); 327 tfCmdBuilder.append(" ./tradefed.sh " + mRemoteTradefedDir + configFile.getName()); 328 if (config.getCommandOptions().shouldUseRemoteSandboxMode()) { 329 tfCmdBuilder.append(" --" + CommandOptions.USE_SANDBOX); 330 } 331 tfCmdBuilder.append(" > " + STDOUT_FILE + " 2> " + STDERR_FILE); 332 remoteTfCommand.add("\"" + tfCmdBuilder.toString() + "\""); 333 // Kick off the actual remote run 334 CommandResult resultRemoteExecution = 335 GceManager.remoteSshCommandExecution( 336 info, options, runUtil, 0L, remoteTfCommand.toArray(new String[0])); 337 if (!CommandStatus.SUCCESS.equals(resultRemoteExecution.getStatus())) { 338 CLog.e("Error running the remote command: %s", resultRemoteExecution.getStdout()); 339 currentInvocationListener.invocationFailed( 340 createInvocationFailure( 341 resultRemoteExecution.getStderr(), FailureStatus.INFRA_FAILURE)); 342 return; 343 } 344 // Sleep a bit to let the process start 345 RunUtil.getDefault().sleep(10000L); 346 347 mProtoParser = new ProtoResultParser(currentInvocationListener, context, false, "remote-"); 348 // Print when parsing 349 mProtoParser.setQuiet(false); 350 // Monitor the remote invocation to ensure it's completing. Block until timeout or stops 351 // running. 352 boolean stillRunning = true; 353 try { 354 stillRunning = 355 isStillRunning( 356 currentInvocationListener, configFile, info, options, runUtil, config); 357 } finally { 358 // Fetch the logs for debugging 359 File stdout = 360 fetchRemoteAndLogFile( 361 currentInvocationListener, 362 STDOUT_FILE, 363 STDOUT_FILE, 364 info, 365 options, 366 runUtil); 367 FileUtil.recursiveDelete(stdout); 368 File stderr = 369 fetchRemoteAndLogFile( 370 currentInvocationListener, 371 STDERR_FILE, 372 STDERR_FILE, 373 info, 374 options, 375 runUtil); 376 if (stderr != null && stderr.exists()) { 377 mRemoteConsoleStdErr = FileUtil.readStringFromFile(stderr); 378 FileUtil.recursiveDelete(stderr); 379 } else { 380 mRemoteConsoleStdErr = "Failed to fetch stderr from remote."; 381 } 382 } 383 384 // If not result in progress are reported, parse the full results at the end. 385 if (!config.getCommandOptions().shouldReportModuleProgression()) { 386 fetchAndProcessResults( 387 stillRunning, 388 currentInvocationListener, 389 info, 390 options, 391 runUtil, 392 mRemoteTradefedDir); 393 } else { 394 if (!mProtoParser.invocationEndedReached()) { 395 String message = 396 String.format( 397 "Parsing of results protos might be incomplete: invocation ended " 398 + "of remote execution was not found. " 399 + TRADEFED_EARLY_TERMINATION, 400 mRemoteConsoleStdErr); 401 currentInvocationListener.invocationFailed( 402 createInvocationFailure(message, FailureStatus.INFRA_FAILURE)); 403 } 404 } 405 } 406 isStillRunning( ITestInvocationListener currentInvocationListener, File configFile, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, IConfiguration config)407 private boolean isStillRunning( 408 ITestInvocationListener currentInvocationListener, 409 File configFile, 410 GceAvdInfo info, 411 TestDeviceOptions options, 412 IRunUtil runUtil, 413 IConfiguration config) 414 throws IOException { 415 long maxTimeout = config.getCommandOptions().getInvocationTimeout(); 416 Long endTime = null; 417 if (maxTimeout > 0L) { 418 endTime = System.currentTimeMillis() + maxTimeout; 419 } 420 boolean stillRunning = true; 421 int errorConnectCount = 0; 422 int currentIndex = 0; 423 long currentTimeOnProto = 0L; 424 while (stillRunning) { 425 if (config.getCommandOptions().shouldReportModuleProgression()) { 426 File resultFile = 427 RemoteFileUtil.fetchRemoteFile( 428 info, 429 options, 430 runUtil, 431 PULL_RESULT_TIMEOUT, 432 mRemoteTradefedDir + PROTO_RESULT_NAME + currentIndex); 433 if (resultFile != null) { 434 currentIndex++; 435 currentTimeOnProto = System.currentTimeMillis(); 436 try { 437 mProtoParser.processFileProto(resultFile); 438 } finally { 439 FileUtil.deleteFile(resultFile); 440 } 441 // Don't sleep in that case since we might have more file to process, this will 442 // sleep next time we don't find a file to process on the remote. 443 continue; 444 } 445 } 446 if (System.currentTimeMillis() - currentTimeOnProto > 7200000) { // 2 hours 447 // If we are stuck on waiting the same proto for over 2 hours, collect some logs 448 File stdout = 449 fetchRemoteAndLogFile( 450 currentInvocationListener, 451 STDOUT_FILE, 452 STDOUT_FILE + "-early", 453 info, 454 options, 455 runUtil); 456 FileUtil.recursiveDelete(stdout); 457 currentTimeOnProto = System.currentTimeMillis(); 458 } 459 460 CommandResult psRes = 461 GceManager.remoteSshCommandExecution( 462 info, 463 options, 464 runUtil, 465 120000L, 466 "ps", 467 "-ef", 468 "| grep", 469 CommandRunner.class.getCanonicalName()); 470 if (!CommandStatus.SUCCESS.equals(psRes.getStatus())) { 471 errorConnectCount++; 472 // If we get several connection errors in a row, give up. 473 if (errorConnectCount > MAX_CONNECTION_REFUSED_COUNT) { 474 CLog.e("Failed to connect to the remote to check running status."); 475 return false; 476 } 477 } else { 478 // Reset the error count 479 errorConnectCount = 0; 480 CLog.d("ps -ef: stdout: %s\nstderr: %s\n", psRes.getStdout(), psRes.getStderr()); 481 stillRunning = psRes.getStdout().contains(configFile.getName()); 482 CLog.d("still running: %s", stillRunning); 483 if (endTime != null && System.currentTimeMillis() > endTime) { 484 currentInvocationListener.invocationFailed( 485 createInvocationFailure( 486 String.format( 487 "Remote invocation timeout after %s", 488 TimeUtil.formatElapsedTime(maxTimeout)), 489 FailureStatus.TIMED_OUT)); 490 break; 491 } 492 } 493 if (stillRunning) { 494 RunUtil.getDefault().sleep(REMOTE_PROCESS_RUNNING_WAIT); 495 } 496 } 497 498 File resultFile = null; 499 if (config.getCommandOptions().shouldReportModuleProgression()) { 500 // Process all remaining proto files available 501 do { 502 resultFile = 503 RemoteFileUtil.fetchRemoteFile( 504 info, 505 options, 506 runUtil, 507 PULL_RESULT_TIMEOUT, 508 mRemoteTradefedDir + PROTO_RESULT_NAME + currentIndex); 509 if (resultFile != null) { 510 currentIndex++; 511 try { 512 mProtoParser.processFileProto(resultFile); 513 } finally { 514 FileUtil.deleteFile(resultFile); 515 } 516 } 517 } while (resultFile != null); 518 } 519 return stillRunning; 520 } 521 522 /** Returns the main remote working directory. */ getRemoteMainDir(TestDeviceOptions options)523 private String getRemoteMainDir(TestDeviceOptions options) { 524 return REMOTE_USER_DIR.replace("{$USER}", options.getInstanceUser()); 525 } 526 527 /** 528 * Sometimes remote adb version is a bit weird and is not running properly the first time. Try 529 * it out once to ensure it starts. 530 */ resetAdb(GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil)531 private void resetAdb(GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil) { 532 CommandResult probAdb = 533 GceManager.remoteSshCommandExecution( 534 info, options, runUtil, 120000L, mRemoteAdbPath, "devices"); 535 CLog.d("remote adb prob: %s", probAdb.getStdout()); 536 CLog.d("%s", probAdb.getStderr()); 537 538 CommandResult versionAdb = 539 GceManager.remoteSshCommandExecution( 540 info, options, runUtil, 120000L, mRemoteAdbPath, "version"); 541 CLog.d("version adb: %s", versionAdb.getStdout()); 542 CLog.d("%s", versionAdb.getStderr()); 543 } 544 545 /** 546 * Remote invocation relies on the adb of the remote, so always collect its logs to make sure we 547 * can debug it appropriately. 548 */ collectAdbLogs( GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, ITestLogger logger)549 private void collectAdbLogs( 550 GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, ITestLogger logger) { 551 CommandResult tmpDirFolder = 552 GceManager.remoteSshCommandExecution( 553 info, options, runUtil, 120000L, "bash -c \"echo \\$TMPDIR\""); 554 String folder = tmpDirFolder.getStdout().trim(); 555 CLog.d("Remote TMPDIR folder is: %s", folder); 556 if (Strings.isNullOrEmpty(folder)) { 557 // If TMPDIR is not set, default to /tmp/ location. 558 folder = "/tmp"; 559 } 560 CommandResult uid = 561 GceManager.remoteSshCommandExecution( 562 info, options, new RunUtil(), 120000L, "bash -c \"echo \\$UID\""); 563 String uidString = uid.getStdout().trim(); 564 CLog.d("Remote $UID for adb is: %s", uidString); 565 566 if (Strings.isNullOrEmpty(uidString)) { 567 CLog.w("Could not determine adb log path."); 568 return; 569 } 570 571 GceManager.logNestedRemoteFile( 572 logger, 573 info, 574 options, 575 runUtil, 576 folder + "/adb." + uidString + ".log", 577 LogDataType.TEXT, 578 "full_adb.log"); 579 } 580 581 /** 582 * Create the configuration that will run in the remote VM. 583 * 584 * @param config The main {@link IConfiguration}. 585 * @param logger A logger where to save the XML configuration for debugging. 586 * @param resultDirPath the remote result dir where results should be saved. 587 * @return A file containing the dumped remote XML configuration. 588 * @throws IOException 589 */ 590 @VisibleForTesting createRemoteConfig(IConfiguration config, ITestLogger logger, String resultDirPath)591 File createRemoteConfig(IConfiguration config, ITestLogger logger, String resultDirPath) 592 throws IOException, ConfigurationException { 593 // Setup the remote reporting to a proto file 594 List<ITestInvocationListener> reporters = new ArrayList<>(); 595 FileProtoResultReporter protoReporter = new FileProtoResultReporter(); 596 OptionSetter protoResSetter = new OptionSetter(protoReporter); 597 if (config.getCommandOptions().shouldReportModuleProgression()) { 598 protoResSetter.setOptionValue( 599 FileProtoResultReporter.PERIODIC_PROTO_WRITING_OPTION, "true"); 600 } 601 protoResSetter.setOptionValue( 602 FileProtoResultReporter.PROTO_OUTPUT_FILE, 603 new File(resultDirPath + PROTO_RESULT_NAME).getPath()); 604 reporters.add(protoReporter); 605 606 config.setTestInvocationListeners(reporters); 607 608 for (IDeviceConfiguration deviceConfig : config.getDeviceConfig()) { 609 deviceConfig.getDeviceRequirements().setSerial(); 610 if (deviceConfig.getDeviceRequirements() instanceof DeviceSelectionOptions) { 611 ((DeviceSelectionOptions) deviceConfig.getDeviceRequirements()) 612 .setDeviceTypeRequested(null); 613 } 614 } 615 616 if (config.getCommandOptions().getShardCount() != null 617 && config.getCommandOptions().getShardIndex() == null) { 618 config.getCommandOptions().setReplicateSetup(true); 619 } 620 621 // Mark the remote invocation as subprocess 622 config.getCommandOptions() 623 .getInvocationData() 624 .put(SubprocessTfLauncher.SUBPROCESS_TAG_NAME, "true"); 625 626 // Unset remote-tf-version to avoid re-downloading from remote VM. 627 OptionSetter deviceOptions = 628 new OptionSetter(config.getDeviceConfig().get(0).getDeviceOptions()); 629 deviceOptions.setOptionValue(TestDeviceOptions.REMOTE_TF_VERSION_OPTION, ""); 630 631 // Dump and log the configuration 632 File configFile = FileUtil.createTempFile(config.getName(), ".xml"); 633 config.dumpXml( 634 new PrintWriter(configFile), 635 new ArrayList<String>(), 636 /* print deprecated */ true, 637 /* print unchanged*/ false); 638 try (InputStreamSource source = new FileInputStreamSource(configFile)) { 639 logger.testLog(REMOTE_CONFIG, LogDataType.XML, source); 640 } 641 return configFile; 642 } 643 644 /** Returns the Tradefed version that should be pushed to the remote to drive the invocation. */ getLocalTradefedPath(ITestInvocationListener listener, File remoteTf)645 private File getLocalTradefedPath(ITestInvocationListener listener, File remoteTf) { 646 if (remoteTf != null && remoteTf.exists()) { 647 return remoteTf; 648 } 649 650 String tfPath = System.getProperty("TF_JAR_DIR"); 651 if (tfPath == null) { 652 listener.invocationFailed( 653 createInvocationFailure( 654 "Failed to find $TF_JAR_DIR.", FailureStatus.INFRA_FAILURE)); 655 return null; 656 } 657 File currentTf = new File(tfPath).getAbsoluteFile(); 658 if (tfPath.equals(".")) { 659 currentTf = new File("").getAbsoluteFile(); 660 } 661 return currentTf; 662 } 663 fetchAndProcessResults( boolean wasStillRunning, ITestInvocationListener invocationListener, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, String resultDirPath)664 private void fetchAndProcessResults( 665 boolean wasStillRunning, 666 ITestInvocationListener invocationListener, 667 GceAvdInfo info, 668 TestDeviceOptions options, 669 IRunUtil runUtil, 670 String resultDirPath) 671 throws InvalidProtocolBufferException, IOException { 672 File resultFile = null; 673 if (wasStillRunning) { 674 CLog.d("Remote invocation was still running. No result can be pulled."); 675 return; 676 } 677 resultFile = 678 RemoteFileUtil.fetchRemoteFile( 679 info, 680 options, 681 runUtil, 682 PULL_RESULT_TIMEOUT, 683 resultDirPath + PROTO_RESULT_NAME); 684 if (resultFile == null) { 685 invocationListener.invocationFailed( 686 createInvocationFailure( 687 String.format( 688 "Could not find remote result file at %s. " 689 + TRADEFED_EARLY_TERMINATION, 690 resultDirPath + PROTO_RESULT_NAME, 691 mRemoteConsoleStdErr), 692 FailureStatus.INFRA_FAILURE)); 693 return; 694 } 695 CLog.d("Fetched remote result file!"); 696 // Report result to listener. 697 try { 698 mProtoParser.processFinalizedProto(TestRecordProtoUtil.readFromFile(resultFile)); 699 } finally { 700 FileUtil.deleteFile(resultFile); 701 } 702 } 703 fetchRemoteAndLogFile( ITestLogger logger, String fileName, String logName, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil)704 private File fetchRemoteAndLogFile( 705 ITestLogger logger, 706 String fileName, 707 String logName, 708 GceAvdInfo info, 709 TestDeviceOptions options, 710 IRunUtil runUtil) { 711 File file = 712 RemoteFileUtil.fetchRemoteFile( 713 info, options, runUtil, PULL_RESULT_TIMEOUT, mRemoteTradefedDir + fileName); 714 if (file != null) { 715 try (InputStreamSource source = new FileInputStreamSource(file, false)) { 716 logger.testLog(logName, LogDataType.TEXT, source); 717 } 718 } 719 return file; 720 } 721 createInvocationFailure(String errorMessage, FailureStatus status)722 private FailureDescription createInvocationFailure(String errorMessage, FailureStatus status) { 723 FailureDescription failure = FailureDescription.create(errorMessage); 724 failure.setFailureStatus(status); 725 failure.setCause(new RuntimeException(errorMessage)); 726 return failure; 727 } 728 createInvocationFailure(Exception e, FailureStatus status)729 private FailureDescription createInvocationFailure(Exception e, FailureStatus status) { 730 FailureDescription failure = FailureDescription.create(e.getMessage()); 731 failure.setFailureStatus(status); 732 failure.setCause(e); 733 return failure; 734 } 735 } 736