1 /*
2  * Copyright (C) 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.tools.flicker.rules
18 
19 import android.platform.test.rule.TestWatcher
20 import android.tools.FLICKER_TAG
21 import android.tools.flicker.FlickerConfig
22 import android.tools.flicker.FlickerService
23 import android.tools.flicker.FlickerServiceResultsCollector
24 import android.tools.flicker.FlickerServiceTracesCollector
25 import android.tools.flicker.IFlickerServiceResultsCollector
26 import android.tools.flicker.annotation.FlickerTest
27 import android.tools.flicker.assertions.AssertionResult
28 import android.tools.flicker.config.FlickerConfig
29 import android.tools.flicker.config.FlickerServiceConfig
30 import android.tools.flicker.config.ScenarioId
31 import android.tools.traces.getDefaultFlickerOutputDir
32 import android.util.Log
33 import androidx.test.platform.app.InstrumentationRegistry
34 import com.google.common.truth.Truth
35 import org.junit.AssumptionViolatedException
36 import org.junit.runner.Description
37 import org.junit.runner.notification.Failure
38 
39 /**
40  * A test rule that runs Flicker as a Service on the tests this rule is applied to.
41  *
42  * Note there are performance implications to using this test rule in tests. Tracing will be enabled
43  * during the test which will slow down everything. So if the test is performance critical then an
44  * alternative should be used.
45  *
46  * @see TODO for examples on how to use this test rule in your own tests
47  */
48 open class FlickerServiceRule
49 @JvmOverloads
50 constructor(
51     enabled: Boolean = true,
52     failTestOnFlicker: Boolean = enabled,
53     failTestOnServiceError: Boolean = false,
54     config: FlickerConfig = FlickerConfig().use(FlickerServiceConfig.DEFAULT),
55     private val metricsCollector: IFlickerServiceResultsCollector =
56         FlickerServiceResultsCollector(
57             flickerService = FlickerService(config),
58             tracesCollector = FlickerServiceTracesCollector(getDefaultFlickerOutputDir()),
59             instrumentation = InstrumentationRegistry.getInstrumentation()
60         ),
61 ) : TestWatcher() {
62     private val enabled: Boolean =
<lambda>null63         InstrumentationRegistry.getArguments().getString("faas:enabled")?.let { it.toBoolean() }
64             ?: enabled
65 
66     private val failTestOnFlicker: Boolean =
<lambda>null67         InstrumentationRegistry.getArguments().getString("faas:failTestOnFlicker")?.let {
68             it.toBoolean()
69         }
70             ?: failTestOnFlicker
71 
72     private val failTestOnServiceError: Boolean =
<lambda>null73         InstrumentationRegistry.getArguments().getString("faas:failTestOnServiceError")?.let {
74             it.toBoolean()
75         }
76             ?: failTestOnServiceError
77 
78     private var testFailed = false
79     private var testSkipped = false
80 
81     /** Invoked when a test is about to start */
startingnull82     public override fun starting(description: Description) {
83         if (shouldRun(description)) {
84             handleStarting(description)
85         }
86     }
87 
88     /** Invoked when a test succeeds */
succeedednull89     public override fun succeeded(description: Description) {
90         if (shouldRun(description)) {
91             handleSucceeded(description)
92         }
93     }
94 
95     /** Invoked when a test fails */
failednull96     public override fun failed(e: Throwable?, description: Description) {
97         if (shouldRun(description)) {
98             handleFailed(e, description)
99         }
100     }
101 
102     /** Invoked when a test is skipped due to a failed assumption. */
skippednull103     public override fun skipped(e: AssumptionViolatedException, description: Description) {
104         if (shouldRun(description)) {
105             handleSkipped(e, description)
106         }
107     }
108 
109     /** Invoked when a test method finishes (whether passing or failing) */
finishednull110     public override fun finished(description: Description) {
111         if (shouldRun(description)) {
112             handleFinished(description)
113         }
114     }
115 
handleStartingnull116     private fun handleStarting(description: Description) {
117         Log.i(LOG_TAG, "Test starting $description")
118         metricsCollector.testStarted(description)
119         testFailed = false
120         testSkipped = false
121     }
122 
handleSucceedednull123     private fun handleSucceeded(description: Description) {
124         Log.i(LOG_TAG, "Test succeeded $description")
125     }
126 
handleFailednull127     private fun handleFailed(e: Throwable?, description: Description) {
128         Log.e(LOG_TAG, "$description test failed with", e)
129         metricsCollector.testFailure(Failure(description, e))
130         testFailed = true
131     }
132 
handleSkippednull133     private fun handleSkipped(e: AssumptionViolatedException, description: Description) {
134         Log.i(LOG_TAG, "Test skipped $description with", e)
135         metricsCollector.testSkipped(description)
136         testSkipped = true
137     }
138 
shouldRunnull139     private fun shouldRun(description: Description): Boolean {
140         // Only run FaaS if test rule is enabled and on tests with FlickerTest annotation if it's
141         // used within the class, otherwise run on all tests
142         if (!enabled) {
143             return false
144         }
145 
146         if (description.annotations.none { it is FlickerTest }) {
147             // FlickerTest annotation is not used within the test class, so run on all tests
148             return true
149         }
150 
151         return testClassHasFlickerTestAnnotations(description.testClass)
152     }
153 
testClassHasFlickerTestAnnotationsnull154     private fun testClassHasFlickerTestAnnotations(testClass: Class<*>): Boolean {
155         return testClass.methods.flatMap { it.annotations.asList() }.any { it is FlickerTest }
156     }
157 
handleFinishednull158     private fun handleFinished(description: Description) {
159         Log.i(LOG_TAG, "Test finished $description")
160         metricsCollector.testFinished(description)
161         for (executionError in metricsCollector.executionErrors) {
162             Log.e(LOG_TAG, "FaaS reported execution errors", executionError)
163         }
164 
165         if (failTestOnServiceError && testContainsServiceError()) {
166             throw metricsCollector.executionErrors.first()
167         }
168 
169         if (testSkipped || testFailed || metricsCollector.executionErrors.isNotEmpty()) {
170             // If we had an execution error or the underlying test failed or was skipped, then we
171             // have no guarantees about the correctness of the flicker assertions and detect
172             // scenarios, so we should not check those and instead return immediately.
173             return
174         }
175 
176         val failedMetrics =
177             metricsCollector.resultsForTest(description).filter {
178                 it.status == AssertionResult.Status.FAIL
179             }
180         val assertionErrors = failedMetrics.flatMap { it.assertionErrors }
181         assertionErrors.forEach {
182             Log.e(LOG_TAG, "FaaS reported an assertion failure:")
183             Log.e(LOG_TAG, it.message)
184             Log.e(LOG_TAG, it.stackTraceToString())
185         }
186 
187         if (failTestOnFlicker && testContainsFlicker(description)) {
188             throw assertionErrors.firstOrNull() ?: error("Unexpectedly missing assertion error")
189         }
190 
191         val flickerTestAnnotation: FlickerTest? =
192             description.annotations.filterIsInstance<FlickerTest>().firstOrNull()
193         if (failTestOnFlicker && flickerTestAnnotation != null) {
194             val detectedScenarios = metricsCollector.detectedScenariosForTest(description)
195             Truth.assertThat(detectedScenarios)
196                 .containsAtLeastElementsIn(flickerTestAnnotation.expected.map { ScenarioId(it) })
197         }
198     }
199 
testContainsFlickernull200     private fun testContainsFlicker(description: Description): Boolean {
201         val resultsForTest = metricsCollector.resultsForTest(description)
202         return resultsForTest.any { it.status == AssertionResult.Status.FAIL }
203     }
204 
testContainsServiceErrornull205     private fun testContainsServiceError(): Boolean {
206         return metricsCollector.executionErrors.isNotEmpty()
207     }
208 
209     companion object {
210         const val LOG_TAG = "$FLICKER_TAG-ServiceRule"
211     }
212 }
213