1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.invoker;
17 
18 import com.android.ddmlib.IDevice;
19 import com.android.ddmlib.Log.LogLevel;
20 import com.android.tradefed.build.BuildRetrievalError;
21 import com.android.tradefed.build.IBuildInfo;
22 import com.android.tradefed.command.CommandRunner.ExitCode;
23 import com.android.tradefed.config.GlobalConfiguration;
24 import com.android.tradefed.config.IConfiguration;
25 import com.android.tradefed.device.DeviceNotAvailableException;
26 import com.android.tradefed.device.DeviceUnresponsiveException;
27 import com.android.tradefed.device.ITestDevice;
28 import com.android.tradefed.device.ITestDevice.RecoveryMode;
29 import com.android.tradefed.device.StubDevice;
30 import com.android.tradefed.device.TestDeviceState;
31 import com.android.tradefed.guice.InvocationScope;
32 import com.android.tradefed.invoker.sandbox.SandboxedInvocationExecution;
33 import com.android.tradefed.invoker.shard.ShardBuildCloner;
34 import com.android.tradefed.log.ILeveledLogOutput;
35 import com.android.tradefed.log.ILogRegistry;
36 import com.android.tradefed.log.LogRegistry;
37 import com.android.tradefed.log.LogUtil.CLog;
38 import com.android.tradefed.result.ITestInvocationListener;
39 import com.android.tradefed.result.InputStreamSource;
40 import com.android.tradefed.result.LogDataType;
41 import com.android.tradefed.result.LogSaverResultForwarder;
42 import com.android.tradefed.result.ResultForwarder;
43 import com.android.tradefed.sandbox.SandboxInvocationRunner;
44 import com.android.tradefed.targetprep.BuildError;
45 import com.android.tradefed.targetprep.DeviceFailedToBootError;
46 import com.android.tradefed.targetprep.TargetSetupError;
47 import com.android.tradefed.testtype.IRemoteTest;
48 import com.android.tradefed.testtype.IResumableTest;
49 import com.android.tradefed.testtype.IRetriableTest;
50 import com.android.tradefed.util.IRunUtil;
51 import com.android.tradefed.util.RunInterruptedException;
52 import com.android.tradefed.util.RunUtil;
53 import com.android.tradefed.util.TimeUtil;
54 
55 import com.google.common.annotations.VisibleForTesting;
56 
57 import java.io.IOException;
58 import java.util.ArrayList;
59 import java.util.Arrays;
60 import java.util.List;
61 import java.util.Map.Entry;
62 import java.util.concurrent.ExecutionException;
63 import java.util.concurrent.TimeUnit;
64 
65 /**
66  * Default implementation of {@link ITestInvocation}.
67  * <p/>
68  * Loads major objects based on {@link IConfiguration}
69  *   - retrieves build
70  *   - prepares target
71  *   - runs tests
72  *   - reports results
73  */
74 public class TestInvocation implements ITestInvocation {
75 
76     /** Key of the command line args attributes */
77     public static final String COMMAND_ARGS_KEY = "command_line_args";
78 
79     /**
80      * Format of the key in {@link IBuildInfo} to log the battery level for each step of the
81      * invocation. (Setup, test, tear down).
82      */
83     private static final String BATTERY_ATTRIBUTE_FORMAT_KEY = "%s-battery-%s";
84 
85     static final String TRADEFED_LOG_NAME = "host_log";
86     static final String DEVICE_LOG_NAME_PREFIX = "device_logcat_";
87     static final String EMULATOR_LOG_NAME_PREFIX = "emulator_log_";
88     static final String BUILD_ERROR_BUGREPORT_NAME = "build_error_bugreport";
89     static final String DEVICE_UNRESPONSIVE_BUGREPORT_NAME = "device_unresponsive_bugreport";
90     static final String INVOCATION_ENDED_BUGREPORT_NAME = "invocation_ended_bugreport";
91     static final String TARGET_SETUP_ERROR_BUGREPORT_NAME = "target_setup_error_bugreport";
92     static final String BATT_TAG = "[battery level]";
93 
94     public enum Stage {
95         ERROR("error"),
96         SETUP("setup"),
97         TEST("test"),
98         TEARDOWN("teardown");
99 
100         private final String mName;
101 
Stage(String name)102         Stage(String name) {
103             mName = name;
104         }
105 
getName()106         public String getName() {
107             return mName;
108         }
109     }
110 
111     private String mStatus = "(not invoked)";
112     private boolean mStopRequested = false;
113 
114     /**
115      * A {@link ResultForwarder} for forwarding resumed invocations.
116      * <p/>
117      * It filters the invocationStarted event for the resumed invocation, and sums the invocation
118      * elapsed time
119      */
120     private static class ResumeResultForwarder extends ResultForwarder {
121 
122         long mCurrentElapsedTime;
123 
124         /**
125          * @param listeners
126          */
ResumeResultForwarder(List<ITestInvocationListener> listeners, long currentElapsedTime)127         public ResumeResultForwarder(List<ITestInvocationListener> listeners,
128                 long currentElapsedTime) {
129             super(listeners);
130             mCurrentElapsedTime = currentElapsedTime;
131         }
132 
133         @Override
invocationStarted(IInvocationContext context)134         public void invocationStarted(IInvocationContext context) {
135             // ignore
136         }
137 
138         @Override
invocationEnded(long newElapsedTime)139         public void invocationEnded(long newElapsedTime) {
140             super.invocationEnded(mCurrentElapsedTime + newElapsedTime);
141         }
142     }
143 
144     /**
145      * Display a log message informing the user of a invocation being started.
146      *
147      * @param context the {@link IInvocationContext}
148      * @param config the {@link IConfiguration}
149      */
logStartInvocation(IInvocationContext context, IConfiguration config)150     private void logStartInvocation(IInvocationContext context, IConfiguration config) {
151         String shardSuffix = "";
152         if (config.getCommandOptions().getShardIndex() != null) {
153             shardSuffix =
154                     String.format(
155                             " (shard %d of %d)",
156                             config.getCommandOptions().getShardIndex() + 1,
157                             config.getCommandOptions().getShardCount());
158         }
159         StringBuilder buildInfos = new StringBuilder();
160         StringBuilder msg = new StringBuilder("Starting invocation for '");
161         msg.append(context.getTestTag());
162         msg.append("' with ");
163         for (Entry<ITestDevice, IBuildInfo> entry : context.getDeviceBuildMap().entrySet()) {
164             msg.append("'[ ");
165             msg.append(entry.getValue().toString());
166             buildInfos.append(entry.getValue().toString());
167             msg.append(" on device '");
168             msg.append(entry.getKey().getSerialNumber());
169             msg.append("'] ");
170         }
171         msg.append(shardSuffix);
172         CLog.logAndDisplay(LogLevel.INFO, msg.toString());
173         mStatus = String.format("running %s on build(s) '%s'", context.getTestTag(),
174                 buildInfos.toString()) + shardSuffix;
175     }
176 
177     /**
178      * Performs the invocation
179      *
180      * @param config the {@link IConfiguration}
181      * @param context the {@link IInvocationContext} to use.
182      */
performInvocation( IConfiguration config, IInvocationContext context, IInvocationExecution invocationPath, IRescheduler rescheduler, ITestInvocationListener listener)183     private void performInvocation(
184             IConfiguration config,
185             IInvocationContext context,
186             IInvocationExecution invocationPath,
187             IRescheduler rescheduler,
188             ITestInvocationListener listener)
189             throws Throwable {
190 
191         boolean resumed = false;
192         String bugreportName = null;
193         long startTime = System.currentTimeMillis();
194         long elapsedTime = -1;
195         Throwable exception = null;
196         Throwable tearDownException = null;
197         ITestDevice badDevice = null;
198 
199         startInvocation(config, context, listener);
200         // Ensure that no unexpected attributes are added afterward
201         ((InvocationContext) context).lockAttributes();
202         try {
203             logDeviceBatteryLevel(context, "initial");
204             prepareAndRun(config, context, invocationPath, listener);
205         } catch (BuildError e) {
206             exception = e;
207             CLog.w("Build failed on device '%s'. Reason: %s", e.getDeviceDescriptor(),
208                     e.toString());
209             bugreportName = BUILD_ERROR_BUGREPORT_NAME;
210             badDevice = context.getDeviceBySerial(e.getDeviceDescriptor().getSerial());
211             if (e instanceof DeviceFailedToBootError) {
212                 if (badDevice == null) {
213                     context.setRecoveryModeForAllDevices(RecoveryMode.NONE);
214                 } else {
215                     badDevice.setRecoveryMode(RecoveryMode.NONE);
216                 }
217             }
218             reportFailure(e, listener, config, context, rescheduler, invocationPath);
219         } catch (TargetSetupError e) {
220             exception = e;
221             CLog.e("Caught exception while running invocation");
222             CLog.e(e);
223             bugreportName = TARGET_SETUP_ERROR_BUGREPORT_NAME;
224             badDevice = context.getDeviceBySerial(e.getDeviceDescriptor().getSerial());
225             reportFailure(e, listener, config, context, rescheduler, invocationPath);
226         } catch (DeviceNotAvailableException e) {
227             exception = e;
228             // log a warning here so its captured before reportLogs is called
229             CLog.w("Invocation did not complete due to device %s becoming not available. " +
230                     "Reason: %s", e.getSerial(), e.getMessage());
231             badDevice = context.getDeviceBySerial(e.getSerial());
232             if ((e instanceof DeviceUnresponsiveException) && badDevice != null
233                     && TestDeviceState.ONLINE.equals(badDevice.getDeviceState())) {
234                 // under certain cases it might still be possible to grab a bugreport
235                 bugreportName = DEVICE_UNRESPONSIVE_BUGREPORT_NAME;
236             }
237             resumed = resume(config, context, rescheduler, System.currentTimeMillis() - startTime);
238             if (!resumed) {
239                 reportFailure(e, listener, config, context, rescheduler, invocationPath);
240             } else {
241                 CLog.i("Rescheduled failed invocation for resume");
242             }
243             // Upon reaching here after an exception, it is safe to assume that recovery
244             // has already been attempted so we disable it to avoid re-entry during clean up.
245             if (badDevice != null) {
246                 badDevice.setRecoveryMode(RecoveryMode.NONE);
247             }
248             throw e;
249         } catch (RunInterruptedException e) {
250             CLog.w("Invocation interrupted");
251             reportFailure(e, listener, config, context, rescheduler, invocationPath);
252         } catch (AssertionError e) {
253             exception = e;
254             CLog.e("Caught AssertionError while running invocation: %s", e.toString());
255             CLog.e(e);
256             reportFailure(e, listener, config, context, rescheduler, invocationPath);
257         } catch (Throwable t) {
258             exception = t;
259             // log a warning here so its captured before reportLogs is called
260             CLog.e("Unexpected exception when running invocation: %s", t.toString());
261             CLog.e(t);
262             reportFailure(t, listener, config, context, rescheduler, invocationPath);
263             throw t;
264         } finally {
265             for (ITestDevice device : context.getDevices()) {
266                 reportLogs(device, listener, Stage.TEST);
267             }
268             getRunUtil().allowInterrupt(false);
269             if (config.getCommandOptions().takeBugreportOnInvocationEnded() ||
270                     config.getCommandOptions().takeBugreportzOnInvocationEnded()) {
271                 if (bugreportName != null) {
272                     CLog.i("Bugreport to be taken for failure instead of invocation ended.");
273                 } else {
274                     bugreportName = INVOCATION_ENDED_BUGREPORT_NAME;
275                 }
276             }
277             if (bugreportName != null) {
278                 if (badDevice == null) {
279                     for (ITestDevice device : context.getDevices()) {
280                         takeBugreport(device, listener, bugreportName);
281                     }
282                 } else {
283                     // If we have identified a faulty device only take the bugreport on it.
284                     takeBugreport(badDevice, listener, bugreportName);
285                 }
286             }
287             mStatus = "tearing down";
288             try {
289                 invocationPath.doTeardown(context, config, exception);
290             } catch (Throwable e) {
291                 tearDownException = e;
292                 CLog.e("Exception when tearing down invocation: %s", tearDownException.toString());
293                 CLog.e(tearDownException);
294                 if (exception == null) {
295                     // only report when the exception is new during tear down
296                     reportFailure(
297                             tearDownException,
298                             listener,
299                             config,
300                             context,
301                             rescheduler,
302                             invocationPath);
303                 }
304             }
305             mStatus = "done running tests";
306             try {
307                 // Clean up host.
308                 invocationPath.doCleanUp(context, config, exception);
309                 for (ITestDevice device : context.getDevices()) {
310                     reportLogs(device, listener, Stage.TEARDOWN);
311                 }
312                 if (mStopRequested) {
313                     CLog.e(
314                             "====================================================================="
315                                     + "====");
316                     CLog.e(
317                             "Invocation was interrupted due to TradeFed stop, results will be "
318                                     + "affected.");
319                     CLog.e(
320                             "====================================================================="
321                                     + "====");
322                 }
323                 reportHostLog(listener, config.getLogOutput());
324                 elapsedTime = System.currentTimeMillis() - startTime;
325                 if (!resumed) {
326                     listener.invocationEnded(elapsedTime);
327                 }
328             } finally {
329                 invocationPath.cleanUpBuilds(context, config);
330             }
331         }
332         if (tearDownException != null) {
333             // this means a DNAE or RTE has happened during teardown, need to throw
334             // if there was a preceding RTE or DNAE stored in 'exception', it would have already
335             // been thrown before exiting the previous try...catch...finally block
336             throw tearDownException;
337         }
338     }
339 
340     /** Do setup and run the tests */
prepareAndRun( IConfiguration config, IInvocationContext context, IInvocationExecution invocationPath, ITestInvocationListener listener)341     private void prepareAndRun(
342             IConfiguration config,
343             IInvocationContext context,
344             IInvocationExecution invocationPath,
345             ITestInvocationListener listener)
346             throws Throwable {
347         if (config.getCommandOptions().shouldUseSandboxing()) {
348             // TODO: extract in new TestInvocation type.
349             // If the invocation is sandboxed run as a sandbox instead.
350             SandboxInvocationRunner.prepareAndRun(config, context, listener);
351             return;
352         }
353         getRunUtil().allowInterrupt(true);
354         logDeviceBatteryLevel(context, "initial -> setup");
355         invocationPath.doSetup(context, config, listener);
356         logDeviceBatteryLevel(context, "setup -> test");
357         invocationPath.runTests(context, config, listener);
358         logDeviceBatteryLevel(context, "after test");
359     }
360 
361     /**
362      * Starts the invocation.
363      * <p/>
364      * Starts logging, and informs listeners that invocation has been started.
365      *
366      * @param config
367      * @param context
368      */
startInvocation(IConfiguration config, IInvocationContext context, ITestInvocationListener listener)369     private void startInvocation(IConfiguration config, IInvocationContext context,
370             ITestInvocationListener listener) {
371         logStartInvocation(context, config);
372         listener.invocationStarted(context);
373     }
374 
375     /**
376      * Attempt to reschedule the failed invocation to resume where it left off.
377      * <p/>
378      * @see IResumableTest
379      *
380      * @param config
381      * @return <code>true</code> if invocation was resumed successfully
382      */
resume(IConfiguration config, IInvocationContext context, IRescheduler rescheduler, long elapsedTime)383     private boolean resume(IConfiguration config, IInvocationContext context,
384             IRescheduler rescheduler, long elapsedTime) {
385         for (IRemoteTest test : config.getTests()) {
386             if (test instanceof IResumableTest) {
387                 IResumableTest resumeTest = (IResumableTest)test;
388                 if (resumeTest.isResumable()) {
389                     // resume this config if any test is resumable
390                     IConfiguration resumeConfig = config.clone();
391                     // reuse the same build for the resumed invocation
392                     ShardBuildCloner.cloneBuildInfos(resumeConfig, resumeConfig, context);
393 
394                     // create a result forwarder, to prevent sending two invocationStarted events
395                     resumeConfig.setTestInvocationListener(new ResumeResultForwarder(
396                             config.getTestInvocationListeners(), elapsedTime));
397                     resumeConfig.setLogOutput(config.getLogOutput().clone());
398                     resumeConfig.setCommandOptions(config.getCommandOptions().clone());
399                     boolean canReschedule = rescheduler.scheduleConfig(resumeConfig);
400                     if (!canReschedule) {
401                         CLog.i("Cannot reschedule resumed config for build. Cleaning up build.");
402                         for (String deviceName : context.getDeviceConfigNames()) {
403                             resumeConfig.getDeviceConfigByName(deviceName).getBuildProvider()
404                                     .cleanUp(context.getBuildInfo(deviceName));
405                         }
406                     }
407                     // FIXME: is it a bug to return from here, when we may not have completed the
408                     // FIXME: config.getTests iteration?
409                     return canReschedule;
410                 }
411             }
412         }
413         return false;
414     }
415 
reportFailure( Throwable exception, ITestInvocationListener listener, IConfiguration config, IInvocationContext context, IRescheduler rescheduler, IInvocationExecution invocationPath)416     private void reportFailure(
417             Throwable exception,
418             ITestInvocationListener listener,
419             IConfiguration config,
420             IInvocationContext context,
421             IRescheduler rescheduler,
422             IInvocationExecution invocationPath) {
423         // Always report the failure
424         listener.invocationFailed(exception);
425         // Reset the build (if necessary) and decide if we should reschedule the configuration.
426         boolean shouldReschedule =
427                 invocationPath.resetBuildAndReschedule(exception, listener, config, context);
428         if (shouldReschedule) {
429             rescheduleTest(config, rescheduler);
430         }
431     }
432 
rescheduleTest(IConfiguration config, IRescheduler rescheduler)433     private void rescheduleTest(IConfiguration config, IRescheduler rescheduler) {
434         for (IRemoteTest test : config.getTests()) {
435             if (!config.getCommandOptions().isLoopMode() && test instanceof IRetriableTest &&
436                     ((IRetriableTest) test).isRetriable()) {
437                 rescheduler.rescheduleCommand();
438                 return;
439             }
440         }
441     }
442 
reportLogs(ITestDevice device, ITestInvocationListener listener, Stage stage)443     private void reportLogs(ITestDevice device, ITestInvocationListener listener, Stage stage) {
444         if (device == null) {
445             return;
446         }
447         // non stub device
448         if (!(device.getIDevice() instanceof StubDevice)) {
449             try (InputStreamSource logcatSource = device.getLogcat()) {
450                 device.clearLogcat();
451                 String name = getDeviceLogName(stage);
452                 listener.testLog(name, LogDataType.LOGCAT, logcatSource);
453             }
454         }
455         // emulator logs
456         if (device.getIDevice() != null && device.getIDevice().isEmulator()) {
457             try (InputStreamSource emulatorOutput = device.getEmulatorOutput()) {
458                 // TODO: Clear the emulator log
459                 String name = getEmulatorLogName(stage);
460                 listener.testLog(name, LogDataType.TEXT, emulatorOutput);
461             }
462 
463         }
464     }
465 
reportHostLog(ITestInvocationListener listener, ILeveledLogOutput logger)466     private void reportHostLog(ITestInvocationListener listener, ILeveledLogOutput logger) {
467         try (InputStreamSource globalLogSource = logger.getLog()) {
468             listener.testLog(TRADEFED_LOG_NAME, LogDataType.TEXT, globalLogSource);
469         }
470         // once tradefed log is reported, all further log calls for this invocation can get lost
471         // unregister logger so future log calls get directed to the tradefed global log
472         getLogRegistry().unregisterLogger();
473         logger.closeLog();
474     }
475 
takeBugreport( ITestDevice device, ITestInvocationListener listener, String bugreportName)476     private void takeBugreport(
477             ITestDevice device, ITestInvocationListener listener, String bugreportName) {
478         if (device == null) {
479             return;
480         }
481         if (device.getIDevice() instanceof StubDevice) {
482             return;
483         }
484         // logBugreport will report a regular bugreport if bugreportz is not supported.
485         boolean res =
486                 device.logBugreport(
487                         String.format("%s_%s", bugreportName, device.getSerialNumber()), listener);
488         if (!res) {
489             CLog.w("Error when collecting bugreport for device '%s'", device.getSerialNumber());
490         }
491     }
492 
493     /**
494      * Gets the {@link ILogRegistry} to use.
495      * <p/>
496      * Exposed for unit testing.
497      */
getLogRegistry()498     ILogRegistry getLogRegistry() {
499         return LogRegistry.getLogRegistry();
500     }
501 
502     /**
503      * Utility method to fetch the default {@link IRunUtil} singleton
504      * <p />
505      * Exposed for unit testing.
506      */
getRunUtil()507     IRunUtil getRunUtil() {
508         return RunUtil.getDefault();
509     }
510 
511     @Override
toString()512     public String toString() {
513         return mStatus;
514     }
515 
516     /**
517      * Log the battery level of each device in the invocation.
518      *
519      * @param context the {@link IInvocationContext} of the invocation.
520      * @param event a {@link String} describing the context of the logging (initial, setup, etc.).
521      */
522     @VisibleForTesting
logDeviceBatteryLevel(IInvocationContext context, String event)523     void logDeviceBatteryLevel(IInvocationContext context, String event) {
524         for (ITestDevice testDevice : context.getDevices()) {
525             if (testDevice == null) {
526                 continue;
527             }
528             IDevice device = testDevice.getIDevice();
529             if (device == null || device instanceof StubDevice) {
530                 continue;
531             }
532             try {
533                 Integer batteryLevel = device.getBattery(500, TimeUnit.MILLISECONDS).get();
534                 CLog.v("%s - %s - %d%%", BATT_TAG, event, batteryLevel);
535                 context.getBuildInfo(testDevice)
536                         .addBuildAttribute(
537                                 String.format(
538                                         BATTERY_ATTRIBUTE_FORMAT_KEY,
539                                         testDevice.getSerialNumber(),
540                                         event),
541                                 batteryLevel.toString());
542                 continue;
543             } catch (InterruptedException | ExecutionException e) {
544                 // fall through
545             }
546 
547             CLog.v("Failed to get battery level for %s", testDevice.getSerialNumber());
548         }
549     }
550 
551     /**
552      * Invoke {@link IInvocationExecution#fetchBuild(IInvocationContext, IConfiguration,
553      * IRescheduler, ITestInvocationListener)} and handles the output as well as failures.
554      *
555      * @param context the {@link IInvocationContext} of the invocation.
556      * @param config the {@link IConfiguration} of this test run.
557      * @param rescheduler the {@link IRescheduler}, for rescheduling portions of the invocation for
558      *     execution on another resource(s)
559      * @param listener the {@link ITestInvocation} to report build download failures.
560      * @param invocationPath the {@link IInvocationExecution} driving the invocation.
561      * @return True if we successfully downloaded the build, false otherwise.
562      * @throws DeviceNotAvailableException
563      */
invokeFetchBuild( IInvocationContext context, IConfiguration config, IRescheduler rescheduler, ITestInvocationListener listener, IInvocationExecution invocationPath)564     private boolean invokeFetchBuild(
565             IInvocationContext context,
566             IConfiguration config,
567             IRescheduler rescheduler,
568             ITestInvocationListener listener,
569             IInvocationExecution invocationPath)
570             throws DeviceNotAvailableException {
571         try {
572             boolean res = invocationPath.fetchBuild(context, config, rescheduler, listener);
573             if (!res) {
574                 mStatus = "(no build to test)";
575                 rescheduleTest(config, rescheduler);
576                 // Set the exit code to error
577                 setExitCode(ExitCode.NO_BUILD, new BuildRetrievalError("No build found to test."));
578                 return false;
579             }
580             return res;
581         } catch (BuildRetrievalError e) {
582             // report an empty invocation, so this error is sent to listeners
583             startInvocation(config, context, listener);
584             // don't want to use #reportFailure, since that will call buildNotTested
585             listener.invocationFailed(e);
586             for (ITestDevice device : context.getDevices()) {
587                 reportLogs(device, listener, Stage.ERROR);
588             }
589             reportHostLog(listener, config.getLogOutput());
590             listener.invocationEnded(0);
591             return false;
592         }
593     }
594 
595     /** {@inheritDoc} */
596     @Override
invoke( IInvocationContext context, IConfiguration config, IRescheduler rescheduler, ITestInvocationListener... extraListeners)597     public void invoke(
598             IInvocationContext context,
599             IConfiguration config,
600             IRescheduler rescheduler,
601             ITestInvocationListener... extraListeners)
602             throws DeviceNotAvailableException, Throwable {
603         List<ITestInvocationListener> allListeners =
604                 new ArrayList<>(config.getTestInvocationListeners().size() + extraListeners.length);
605         allListeners.addAll(config.getTestInvocationListeners());
606         allListeners.addAll(Arrays.asList(extraListeners));
607         ITestInvocationListener listener =
608                 new LogSaverResultForwarder(config.getLogSaver(), allListeners);
609         IInvocationExecution invocationPath =
610                 createInvocationExec(config.getConfigurationDescription().shouldUseSandbox());
611 
612         // Create the Guice scope
613         InvocationScope scope = getInvocationScope();
614         scope.enter();
615         // Seed our TF objects to the Guice scope
616         scope.seedConfiguration(config);
617         try {
618             mStatus = "fetching build";
619             config.getLogOutput().init();
620             getLogRegistry().registerLogger(config.getLogOutput());
621             for (String deviceName : context.getDeviceConfigNames()) {
622                 context.getDevice(deviceName).clearLastConnectedWifiNetwork();
623                 context.getDevice(deviceName)
624                         .setOptions(config.getDeviceConfigByName(deviceName).getDeviceOptions());
625                 if (config.getDeviceConfigByName(deviceName)
626                         .getDeviceOptions()
627                         .isLogcatCaptureEnabled()) {
628                     if (!(context.getDevice(deviceName).getIDevice() instanceof StubDevice)) {
629                         context.getDevice(deviceName).startLogcat();
630                     }
631                 }
632             }
633 
634             String cmdLineArgs = config.getCommandLine();
635             if (cmdLineArgs != null) {
636                 CLog.i("Invocation was started with cmd: %s", cmdLineArgs);
637             }
638 
639             long start = System.currentTimeMillis();
640             boolean providerSuccess =
641                     invokeFetchBuild(context, config, rescheduler, listener, invocationPath);
642             long fetchBuildDuration = System.currentTimeMillis() - start;
643             context.addInvocationTimingMetric(IInvocationContext.TimingEvent.FETCH_BUILD,
644                     fetchBuildDuration);
645             CLog.d("Fetch build duration: %s", TimeUtil.formatElapsedTime(fetchBuildDuration));
646             if (!providerSuccess) {
647                 return;
648             }
649 
650             mStatus = "sharding";
651             boolean sharding = invocationPath.shardConfig(config, context, rescheduler);
652             if (sharding) {
653                 CLog.i("Invocation for %s has been sharded, rescheduling", context.getSerials());
654                 return;
655             }
656 
657             if (config.getTests() == null || config.getTests().isEmpty()) {
658                 CLog.e("No tests to run");
659                 return;
660             }
661 
662             performInvocation(config, context, invocationPath, rescheduler, listener);
663             setExitCode(ExitCode.NO_ERROR, null);
664         } catch (IOException e) {
665             CLog.e(e);
666         } finally {
667             scope.exit();
668             // Ensure build infos are always cleaned up at the end of invocation.
669             invocationPath.cleanUpBuilds(context, config);
670 
671             // ensure we always deregister the logger
672             for (String deviceName : context.getDeviceConfigNames()) {
673                 if (!(context.getDevice(deviceName).getIDevice() instanceof StubDevice)) {
674                     context.getDevice(deviceName).stopLogcat();
675                 }
676             }
677             // save remaining logs contents to global log
678             getLogRegistry().dumpToGlobalLog(config.getLogOutput());
679             // Ensure log is unregistered and closed
680             getLogRegistry().unregisterLogger();
681             config.getLogOutput().closeLog();
682         }
683     }
684 
685     /** Returns the current {@link InvocationScope}. */
686     @VisibleForTesting
getInvocationScope()687     InvocationScope getInvocationScope() {
688         return InvocationScope.getDefault();
689     }
690 
691     /**
692      * Helper to set the exit code. Exposed for testing.
693      */
setExitCode(ExitCode code, Throwable stack)694     protected void setExitCode(ExitCode code, Throwable stack) {
695         GlobalConfiguration.getInstance().getCommandScheduler()
696                 .setLastInvocationExitCode(code, stack);
697     }
698 
getDeviceLogName(Stage stage)699     public static String getDeviceLogName(Stage stage) {
700         return DEVICE_LOG_NAME_PREFIX + stage.getName();
701     }
702 
getEmulatorLogName(Stage stage)703     public static String getEmulatorLogName(Stage stage) {
704         return EMULATOR_LOG_NAME_PREFIX + stage.getName();
705     }
706 
707     @Override
notifyInvocationStopped()708     public void notifyInvocationStopped() {
709         mStopRequested = true;
710     }
711 
712     /**
713      * Create the invocation path that should be followed.
714      *
715      * @param isSandboxed If we are currently running in the sandbox, then a special path is
716      *     applied.
717      * @return The {@link IInvocationExecution} describing the invocation.
718      */
createInvocationExec(boolean isSandboxed)719     public IInvocationExecution createInvocationExec(boolean isSandboxed) {
720         if (isSandboxed) {
721             return new SandboxedInvocationExecution();
722         }
723         return new InvocationExecution();
724     }
725 }
726