/* * Copyright (C) 2022 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.car.test; import static java.lang.annotation.ElementType.METHOD; import static java.lang.annotation.ElementType.TYPE; import static java.lang.annotation.RetentionPolicy.RUNTIME; import android.annotation.Nullable; import android.car.Car; import android.car.CarVersion; import android.car.PlatformVersion; import android.car.PlatformVersionMismatchException; import android.car.annotation.AddedInOrBefore; import android.car.annotation.ApiRequirements; import android.car.test.ApiCheckerRule.UnsupportedVersionTest.Behavior; import android.os.Build; import android.text.TextUtils; import android.util.Log; import android.util.Pair; import com.android.compatibility.common.util.ApiTest; import com.android.compatibility.common.util.CddTest; import com.android.compatibility.common.util.NonApiTest; import org.junit.AssumptionViolatedException; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; import java.lang.annotation.Annotation; import java.lang.annotation.Retention; import java.lang.annotation.Target; import java.lang.reflect.Field; import java.lang.reflect.Member; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.List; /** * Rule used to validate Car API requirements on CTS tests. * *

This rule is used to verify that all tests in a class: * *

    *
  1. Indicate which API / CDD is being tested. *
  2. Properly behave on supported and unsupported versions. *
* *

For the former, the test must be annotated with either {@link ApiTest} or {@link CddTest} (in * which case it also needs to be annotated with {@link ApiRequirements}, otherwise the test will * fail (unless the rule was created with {@link Builder#disableAnnotationsCheck()}. In the case * of {@link ApiTest}, the rule will also assert that the underlying APIs are annotated with either * {@link ApiRequirements} or {@link AddedInOrBefore}. * *

Note: Usually, all CTS tests should be testing public or system APIs or CDD * requirements. However, in the case that they don't (especially in {@code AndroidCarApiTest}), * they should be annotated with {@link NonApiTest}. This usage should also be justified. * *

For the latter, if the API declares {@link ApiRequirements}, the rule by default will make * sure the test behaves properly in the supported and unsupported platform versions: *

    *
  1. If the platform is supported, the test should pass as usual. *
  2. If the platform is not supported, the rule will assert that the test throws a * {@link PlatformVersionMismatchException}. *
* *

There are corner cases where the default rule behavior cannot be applied for the test, like: *

    *
  1. The test logic is too complex (or takes time) and should be simplified when running on * unsupported versions. *
  2. The API being tested should behave differently on supported or unsupported versions. *
* *

In these cases, the test should be split in 2 tests, one for the supported version and another * for the unsupported version, and annotated with {@link SupportedVersionTest} or * {@link UnsupportedVersionTest} respectively; these tests MUST be provided in pair (in * fact, these annotations take an argument pointing to the pair) and they will behave this way: * *

    *
  1. {@link SupportedVersionTest}: should pass on supported platform and will be ignored on * unsupported platforms (by throwing an {@link ExpectedVersionAssumptionViolationException}). *
  2. {@link UnsupportedVersionTest}: by default, it will be ignored on supported platforms * (by throwing an {@link ExpectedVersionAssumptionViolationException}), but can be changed * to run on unsupported platforms as well (by setting its * {@link UnsupportedVersionTest#behavior()} to {@link Behavior#EXPECT_PASS}. *
* *

So, back to the examples above, the tests would be: * 

{@code
 *
 * @Test
 * @ApiTest(apis = {"com.acme.Car#foo"})
 * @SupportedVersionTest(unsupportedVersionTest="testFoo_unsupported")
 * public void testFoo_supported() {
 *    baz(); // takes a long time
 *    foo();
 * }
 *
 * @Test
 * @ApiTest(apis = {"com.acme.Car#foo"})
 * @UnsupportedVersionTest(supportedVersionTest="testFoo_supported")
 * public void testFoo_unsupported() {
 *    foo(); // should throw PlatformViolationException
 * }
 *
 * @Test
 * @ApiTest(apis = {"com.acme.Car#bar"})
 * @SupportedVersionTest(unsupportedVersionTest="testBar_unsupported")
 * public void testBar_supported() {
 *    assertWithMessage("bar()").that(bar()).isEqualTo("BehaviorOnSupportedPlatform");
 * }
 *
 * @Test
 * @ApiTest(apis = {"com.acme.Car#bar"})
 * @UnsupportedVersionTest(supportedVersionTest="testBar_supported", behavior=EXPECT_PASS)
 * public void testFoo_unsupported() {
 *    assertWithMessage("bar()").that(bar()).isEqualTo("BehaviorOnUnsupportedPlatform");
 * }
 *
 * }
* * For nested classes the following annotation should be used for methods: * 
{@code
 * @Test
 * @ApiTest(apis = {"com.acme.Car$Inner#methodName"})
 * public void testMethodName() {}
 *
 *
* For nested classes the following annotation should be used for fields: * 

 * @Test
 * @ApiTest(apis = {"com.acme.Car.Inner#fieldName"})
 * public void testFieldName() {}
 *
 * }
*/ public final class ApiCheckerRule implements TestRule { public static final String TAG = ApiCheckerRule.class.getSimpleName(); private static final boolean DBG = Log.isLoggable(TAG, Log.DEBUG); private final boolean mEnforceTestApiAnnotations; private final boolean mEnforceApiRequirements; @Nullable private String mTestMethodName; /** * Builder. */ public static final class Builder { private boolean mEnforceTestApiAnnotations = true; private boolean mEnforceApiRequirements = true; /** * Creates a new rule. */ public ApiCheckerRule build() { return new ApiCheckerRule(this); } /** * Don't fail the test if the required annotations (like {@link ApiTest}) are missing. */ public Builder disableAnnotationsCheck() { mEnforceTestApiAnnotations = false; return this; } /** * Don't fail the test if it could not infer its {@link ApiRequirements}. * *

Typically used on tests for built-in APIs. */ public Builder disableApiRequirementsCheck() { mEnforceApiRequirements = false; return this; } } private ApiCheckerRule(Builder builder) { mEnforceTestApiAnnotations = builder.mEnforceTestApiAnnotations; mEnforceApiRequirements = mEnforceTestApiAnnotations ? builder.mEnforceApiRequirements : false; } /** * Checks whether the test is running in an environment that supports the given API. * * @param api API as defined by {@link ApiTest}. * @return whether the test is running in an environment that supports the * {@link ApiRequirements} defined in such API. */ public boolean isApiSupported(String api) { ApiRequirements apiRequirements = getApiRequirements(api); if (apiRequirements == null) { throw new IllegalArgumentException("No @ApiRequirements on " + api); } return isSupported(apiRequirements); } /** * Gets the name of the test being executed. */ @Nullable public String getTestMethodName() { return mTestMethodName; } private boolean isSupported(ApiRequirements apiRequirements) { PlatformVersion platformVersion = Car.getPlatformVersion(); boolean isSupported = platformVersion .isAtLeast(apiRequirements.minPlatformVersion().get()); if (DBG) { Log.d(TAG, "isSupported(" + apiRequirements + "): platformVersion=" + platformVersion + ",supported=" + isSupported); } return isSupported; } private static ApiRequirements getApiRequirements(String api) { Member member = ApiHelper.resolve(api); if (member == null) { throw new IllegalArgumentException("API not found: " + api); } return getApiRequirements(member); } private static ApiRequirements getApiRequirements(Member member) { return getAnnotation(ApiRequirements.class, member); } @SuppressWarnings("deprecation") private static AddedInOrBefore getAddedInOrBefore(Member member) { return getAnnotation(AddedInOrBefore.class, member); } private static T getAnnotation(Class annotationClass, Member member) { if (member instanceof Field) { return ((Field) member).getAnnotation(annotationClass); } if (member instanceof Method) { return ((Method) member).getAnnotation(annotationClass); } throw new UnsupportedOperationException("Invalid member type for API: " + member); } @Override public Statement apply(Statement base, Description description) { return new Statement() { @Override public void evaluate() throws Throwable { base.evaluate(); } }; } // TODO(b/285930588):ApiCheckerRule is no longer required. But the code can be useful, // Currently disabling the rule. As part of the bug, more investigation is required how to // clean up API Checker Rule. public Statement applyOld(Statement base, Description description) { return new Statement() { @Override public void evaluate() throws Throwable { mTestMethodName = description.getMethodName(); try { evaluateInternal(); } finally { mTestMethodName = null; } } private void evaluateInternal() throws Throwable { if (DBG) { Log.d(TAG, "evaluating " + description.getDisplayName()); } // Need to do a basic version check first, as the rule could be used on ATS tests // running on pre-mainline versions if (!isPlatformSupported(description)) { base.evaluate(); return; } // Variables below are used to validate that all ApiRequirements are compatible ApiTest apiTest = null; NonApiTest nonApiTest = null; ApiRequirements apiRequirementsOnApiUnderTest = null; IgnoreInvalidApi ignoreInvalidApi = null; // Optional annotations that change the behavior of the rule SupportedVersionTest supportedVersionTest = null; UnsupportedVersionTest unsupportedVersionTest = null; // Other relevant annotations @SuppressWarnings("deprecation") AddedInOrBefore addedInOrBefore = null; CddTest cddTest = null; ApiRequirements apiRequirementsOnTest = null; // user only with CddTest ApiRequirements effectiveApiRequirementsOnTest = null; for (Annotation annotation : description.getAnnotations()) { if (DBG) { Log.d(TAG, "Annotation: " + annotation); } if (annotation instanceof ApiTest) { apiTest = (ApiTest) annotation; continue; } if (annotation instanceof NonApiTest) { nonApiTest = (NonApiTest) annotation; continue; } if (annotation instanceof ApiRequirements) { apiRequirementsOnTest = (ApiRequirements) annotation; continue; } if (annotation instanceof CddTest) { cddTest = (CddTest) annotation; continue; } if (annotation instanceof SupportedVersionTest) { supportedVersionTest = (SupportedVersionTest) annotation; continue; } if (annotation instanceof UnsupportedVersionTest) { unsupportedVersionTest = (UnsupportedVersionTest) annotation; continue; } if (annotation instanceof IgnoreInvalidApi) { ignoreInvalidApi = (IgnoreInvalidApi) annotation; continue; } } if (DBG) { Log.d(TAG, "Relevant annotations on test: " + "ApiTest=" + apiTest + " CddTest=" + cddTest + " NonApiTest= " + nonApiTest + " ApiRequirements=" + apiRequirementsOnTest + " SupportedVersionTest=" + supportedVersionTest + " UnsupportedVersionTest=" + unsupportedVersionTest + " IgnoreInvalidApi=" + ignoreInvalidApi); } validateOptionalAnnotations(description.getTestClass(), description.getMethodName(), supportedVersionTest, unsupportedVersionTest); if (apiTest == null && (cddTest != null || nonApiTest != null)) { validateNonApiAnnotations(cddTest, nonApiTest, apiRequirementsOnTest); effectiveApiRequirementsOnTest = apiRequirementsOnTest; } if (apiTest == null && cddTest == null && nonApiTest == null) { if (mEnforceTestApiAnnotations) { throw new IllegalArgumentException( "Test is missing @ApiTest, @NonApiTest, or @CddTest annotation"); } else { Log.w(TAG, "Test " + description + " doesn't have @ApiTest, @NonApiTest, or @CddTest," + "but rule is not enforcing it"); } } if (apiTest != null) { Pair pair = getApiRequirementsFromApis( description, apiTest, ignoreInvalidApi); apiRequirementsOnApiUnderTest = pair.first; if (effectiveApiRequirementsOnTest == null) { // not set by CddTest effectiveApiRequirementsOnTest = apiRequirementsOnApiUnderTest; } if (effectiveApiRequirementsOnTest == null && ignoreInvalidApi != null) { effectiveApiRequirementsOnTest = apiRequirementsOnTest; } addedInOrBefore = pair.second; } if (DBG) { Log.d(TAG, "Relevant annotations on APIs: " + "ApiRequirements=" + apiRequirementsOnApiUnderTest + ", AddedInOrBefore: " + addedInOrBefore); } if (apiRequirementsOnApiUnderTest != null && apiRequirementsOnTest != null) { throw new IllegalArgumentException("Test cannot be annotated with both " + "@ApiTest and @ApiRequirements"); } if (effectiveApiRequirementsOnTest == null) { if (ignoreInvalidApi != null) { if (mEnforceTestApiAnnotations) { throw new IllegalArgumentException("Test contains @IgnoreInvalidApi but" + " is missing @ApiRequirements"); } else { Log.w(TAG, "Test " + description + " contains @IgnoreInvalidApi and is " + "missing @ApiRequirements, but rule is not enforcing them"); } } else if (addedInOrBefore == null) { if (mEnforceApiRequirements) { throw new IllegalArgumentException("Missing @ApiRequirements " + "or @AddedInOrBefore"); } else { Log.w(TAG, "Test " + description + " doesn't have required " + "@ApiRequirements or @AddedInOrBefore but rule is not " + "enforcing them"); } } base.evaluate(); return; } // Finally, run the test and assert results depending on whether it's supported or // not apply(base, description, effectiveApiRequirementsOnTest, supportedVersionTest, unsupportedVersionTest); } }; } // apply protected boolean isPlatformSupported(Description description) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { Log.d(TAG, "Running " + description.getDisplayName() + " as-is on pre-TM platform build" + " (" + Build.VERSION.SDK_INT + ")"); return false; } if (!Car.isApiVersionAtLeast(Build.VERSION_CODES.TIRAMISU, /* minor= */ 1)) { Log.d(TAG, "Running " + description.getDisplayName() + " as-is on pre-TM-QPR1 Car build" + " (major=" + Car.getCarVersion().getMajorVersion() + ", minor=" + Car.getCarVersion().getMinorVersion() + ")"); return false; } return true; } private void validateNonApiAnnotations(CddTest cddTest, NonApiTest nonApiTest, ApiRequirements apiRequirements) { if (cddTest != null && nonApiTest != null) { throw new IllegalArgumentException("Test contains both " + nonApiTest.annotationType() + " annotation (" + nonApiTest + ") and " + cddTest.annotationType() + " annotation (" + cddTest + ")"); } if (cddTest != null) { validateCddTestAnnotation(cddTest, apiRequirements); return; } if (nonApiTest != null) { validateNonApiTestAnnotation(nonApiTest, apiRequirements); } } private void validateCddTestAnnotation(CddTest cddTest, ApiRequirements apiRequirements) { @SuppressWarnings("deprecation") String deprecatedRequirement = cddTest.requirement(); if (!TextUtils.isEmpty(deprecatedRequirement)) { throw new IllegalArgumentException("Test contains " + cddTest.annotationType() + " annotation (" + cddTest + "), but it's using the" + " deprecated 'requirement' field (value=" + deprecatedRequirement + "); it " + "should use 'requirements' instead"); } String[] requirements = cddTest.requirements(); if (requirements == null || requirements.length == 0) { throw new IllegalArgumentException("Test contains " + cddTest.annotationType() + " annotation (" + cddTest + "), but it's 'requirements' field is empty (value=" + Arrays.toString(requirements) + ")"); } for (String requirement : requirements) { String trimmedRequirement = requirement == null ? "" : requirement.trim(); if (TextUtils.isEmpty(trimmedRequirement)) { throw new IllegalArgumentException("Test contains " + cddTest.annotationType() + " annotation (" + cddTest + "), but it contains an empty requirement" + "(requirements=" + Arrays.toString(requirements) + ")"); } } // CddTest itself is valid, must have ApiRequirements if (apiRequirements == null) { throw new IllegalArgumentException("Test contains " + cddTest.annotationType() + " annotation (" + cddTest + "), but it's missing @ApiRequirements)"); } } public void validateNonApiTestAnnotation(NonApiTest nonApiTest, ApiRequirements apiRequirements) { if (apiRequirements == null) { throw new IllegalArgumentException("Test contains " + nonApiTest.annotationType() + " annotation (" + nonApiTest + "), but it's missing @ApiRequirements)"); } } @SuppressWarnings("deprecation") private Pair getApiRequirementsFromApis( Description description, ApiTest apiTest, @Nullable IgnoreInvalidApi ignoreInvalidApi) { ApiRequirements firstApiRequirements = null; AddedInOrBefore addedInOrBefore = null; List allApis = new ArrayList<>(); List allApiRequirements = new ArrayList<>(); boolean compatibleApis = true; String[] apis = apiTest.apis(); if (apis == null || apis.length == 0) { throw new IllegalArgumentException("empty @ApiTest annotation"); } List invalidApis = new ArrayList<>(); for (String api : apis) { allApis.add(api); Member member = ApiHelper.resolve(api); if (member == null) { invalidApis.add(api); continue; } ApiRequirements apiRequirements = getApiRequirements(member); if (apiRequirements == null && addedInOrBefore == null) { addedInOrBefore = getAddedInOrBefore(member); if (DBG) { Log.d(TAG, "No @ApiRequirements on " + api + "; trying " + "@AddedInOrBefore instead: " + addedInOrBefore); } continue; } if (apiRequirements == null) { continue; } allApiRequirements.add(apiRequirements); if (firstApiRequirements == null) { firstApiRequirements = apiRequirements; continue; } // Make sure all ApiRequirements are compatible if (!apiRequirements.minCarVersion() .equals(firstApiRequirements.minCarVersion()) || !apiRequirements.minPlatformVersion() .equals(firstApiRequirements.minPlatformVersion())) { Log.w(TAG, "Found incompatible API requirement (" + apiRequirements + ") on " + api + "(first ApiRequirements is " + firstApiRequirements + ")"); compatibleApis = false; } else { Log.d(TAG, "Multiple @ApiRequirements found but they're compatible"); } } if (!invalidApis.isEmpty()) { if (ignoreInvalidApi != null) { Log.i(TAG, "Could not resolve some APIs (" + invalidApis + ") on annotation (" + apiTest + "), but letting it go due to " + ignoreInvalidApi); } else { throw new IllegalArgumentException("Could not resolve some APIs (" + invalidApis + ") on annotation (" + apiTest + ")"); } } else if (!compatibleApis) { throw new IncompatibleApiRequirementsException(allApis, allApiRequirements); } return new Pair<>(firstApiRequirements, addedInOrBefore); } private void validateOptionalAnnotations(Class testClass, String testMethodName, @Nullable SupportedVersionTest supportedVersionAnnotationOnTestMethod, @Nullable UnsupportedVersionTest unsupportedVersionAnnotationOnTestMethod) { if (unsupportedVersionAnnotationOnTestMethod != null && supportedVersionAnnotationOnTestMethod != null) { throw new IllegalArgumentException("test must be annotated with either " + "supportedVersionTest or unsupportedVersionTest, not both"); } if (unsupportedVersionAnnotationOnTestMethod != null) { validateUnsupportedVersionTest(testClass, testMethodName, unsupportedVersionAnnotationOnTestMethod); return; } if (supportedVersionAnnotationOnTestMethod != null) { validateSupportedVersionTest(testClass, testMethodName, supportedVersionAnnotationOnTestMethod); return; } } private void validateUnsupportedVersionTest(Class testClass, String testMethodName, @Nullable UnsupportedVersionTest unsupportedVersionAnnotationOnTestMethod) { // Test class must have a counterpart supportedVersionTest String supportedVersionMethodName = unsupportedVersionAnnotationOnTestMethod .supportedVersionTest(); if (TextUtils.isEmpty(supportedVersionMethodName)) { throw new IllegalArgumentException("missing supportedVersionTest on " + unsupportedVersionAnnotationOnTestMethod); } Method supportedVersionMethod = null; Class[] noParams = {}; try { supportedVersionMethod = testClass.getDeclaredMethod(supportedVersionMethodName, noParams); } catch (Exception e) { Log.w(TAG, "Error getting method named " + supportedVersionMethodName + " on class " + testClass, e); throw new IllegalArgumentException("invalid supportedVersionTest on " + unsupportedVersionAnnotationOnTestMethod + ": " + e); } // And it must be annotated with @SupportedVersionTest SupportedVersionTest supportedVersionAnnotationOnUnsupportedMethod = supportedVersionMethod.getAnnotation(SupportedVersionTest.class); if (supportedVersionAnnotationOnUnsupportedMethod == null) { throw new IllegalArgumentException( "invalid supportedVersionTest method (" + supportedVersionMethodName + " on " + unsupportedVersionAnnotationOnTestMethod + ": it's not annotated with @SupportedVersionTest"); } // which in turn must point to the UnsupportedVersionTest itself String unsupportedVersionMethodOnSupportedAnnotation = supportedVersionAnnotationOnUnsupportedMethod.unsupportedVersionTest(); if (!testMethodName.equals(unsupportedVersionMethodOnSupportedAnnotation)) { throw new IllegalArgumentException( "invalid unsupportedVersionTest on " + supportedVersionAnnotationOnUnsupportedMethod + " annotation on method " + supportedVersionMethodName + ": it should be " + testMethodName); } } private void validateSupportedVersionTest(Class testClass, String testMethodName, @Nullable SupportedVersionTest supportedVersionAnnotationOnTestMethod) { // Test class must have a counterpart unsupportedVersionTest String unsupportedVersionMethodName = supportedVersionAnnotationOnTestMethod .unsupportedVersionTest(); if (TextUtils.isEmpty(unsupportedVersionMethodName)) { throw new IllegalArgumentException("missing unsupportedVersionTest on " + supportedVersionAnnotationOnTestMethod); } Method unsupportedVersionMethod = null; Class[] noParams = {}; try { unsupportedVersionMethod = testClass.getDeclaredMethod(unsupportedVersionMethodName, noParams); } catch (Exception e) { Log.w(TAG, "Error getting method named " + unsupportedVersionMethodName + " on class " + testClass, e); throw new IllegalArgumentException("invalid supportedVersionTest on " + supportedVersionAnnotationOnTestMethod + ": " + e); } // And it must be annotated with @UnupportedVersionTest UnsupportedVersionTest unsupportedVersionAnnotationOnUnsupportedMethod = unsupportedVersionMethod.getAnnotation(UnsupportedVersionTest.class); if (unsupportedVersionAnnotationOnUnsupportedMethod == null) { throw new IllegalArgumentException( "invalid supportedVersionTest method (" + unsupportedVersionMethodName + " on " + supportedVersionAnnotationOnTestMethod + ": it's not annotated with @UnsupportedVersionTest"); } // which in turn must point to the UnsupportedVersionTest itself String supportedVersionMethodOnSupportedAnnotation = unsupportedVersionAnnotationOnUnsupportedMethod.supportedVersionTest(); if (!testMethodName.equals(supportedVersionMethodOnSupportedAnnotation)) { throw new IllegalArgumentException( "invalid supportedVersionTest on " + unsupportedVersionAnnotationOnUnsupportedMethod + " annotation on method " + unsupportedVersionMethodName + ": it should be " + testMethodName); } } private void apply(Statement base, Description description, @Nullable ApiRequirements apiRequirements, @Nullable SupportedVersionTest supportedVersionTest, @Nullable UnsupportedVersionTest unsupportedVersionTest) throws Throwable { if (DBG) { Log.d(TAG, "Applying rule using ApiRequirements=" + apiRequirements); } if (apiRequirements == null) { Log.w(TAG, "No @ApiRequirements on " + description.getDisplayName() + " (most likely it's annotated with @AddedInOrBefore), running it always"); base.evaluate(); return; } if (isSupported(apiRequirements)) { applyOnSupportedVersion(base, description, apiRequirements, unsupportedVersionTest); return; } applyOnUnsupportedVersion(base, description, apiRequirements, supportedVersionTest, unsupportedVersionTest); } private void applyOnSupportedVersion(Statement base, Description description, ApiRequirements apiRequirements, @Nullable UnsupportedVersionTest unsupportedVersionTest) throws Throwable { if (unsupportedVersionTest == null) { if (DBG) { Log.d(TAG, "Car / Platform combo is supported, running " + description.getDisplayName()); } base.evaluate(); return; } Log.i(TAG, "Car / Platform combo IS supported, but ignoring " + description.getDisplayName() + " because it's annotated with " + unsupportedVersionTest); throw new ExpectedVersionAssumptionViolationException(unsupportedVersionTest, Car.getCarVersion(), Car.getPlatformVersion(), apiRequirements); } private void applyOnUnsupportedVersion(Statement base, Description description, ApiRequirements apiRequirements, @Nullable SupportedVersionTest supportedVersionTest, @Nullable UnsupportedVersionTest unsupportedVersionTest) throws Throwable { Behavior behavior = unsupportedVersionTest == null ? null : unsupportedVersionTest.behavior(); if (supportedVersionTest == null && !Behavior.EXPECT_PASS.equals(behavior)) { Log.i(TAG, "Car / Platform combo is NOT supported, running " + description.getDisplayName() + " but expecting " + "PlatformVersionMismatchException"); try { base.evaluate(); throw new PlatformVersionMismatchExceptionNotThrownException( Car.getCarVersion(), Car.getPlatformVersion(), apiRequirements); } catch (PlatformVersionMismatchException e) { if (DBG) { Log.d(TAG, "Exception thrown as expected: " + e); } } return; } if (supportedVersionTest != null) { Log.i(TAG, "Car / Platform combo is NOT supported, but ignoring " + description.getDisplayName() + " because it's annotated with " + supportedVersionTest); throw new ExpectedVersionAssumptionViolationException(supportedVersionTest, Car.getCarVersion(), Car.getPlatformVersion(), apiRequirements); } // At this point, it's annotated with RUN_ALWAYS Log.i(TAG, "Car / Platform combo is NOT supported but running anyways becaucase test is" + " annotated with " + unsupportedVersionTest); base.evaluate(); } /** * Defines the behavior of a test when it's run in an unsupported device (when it's run in a * supported device, the rule will throw a {@link ExpectedVersionAssumptionViolationException} * exception). * *

Without this annotation, a test is expected to throw a * {@link PlatformVersionMismatchException} when running in an unsupported version. * *

Note: a test annotated with this annotation MUST have a counterpart test * annotated with {@link SupportedVersionTest}. */ @Retention(RUNTIME) @Target({TYPE, METHOD}) public @interface UnsupportedVersionTest { /** * Name of the counterpart test should be run on supported versions; such test must be * annoted with {@link SupportedVersionTest}, whith its {@code unsupportedVersionTest} * value point to the test being annotated with this annotation. */ String supportedVersionTest(); /** * Behavior of the test when it's run on unsupported versions. */ Behavior behavior() default Behavior.EXPECT_THROWS_VERSION_MISMATCH_EXCEPTION; @SuppressWarnings("Enum") enum Behavior { /** * Rule will run the test and assert it throws a * {@link PlatformVersionMismatchException}. */ EXPECT_THROWS_VERSION_MISMATCH_EXCEPTION, /** Rule will run the test and assume it will pass.*/ EXPECT_PASS } } /** * Defines a test to be a counterpart of a test annotated with {@link UnsupportedVersionTest}. * *

Such test will be run as usual on supported devices, but will throw a * {@link ExpectedVersionAssumptionViolationException} when running on unsupported devices. * */ @Retention(RUNTIME) @Target({TYPE, METHOD}) public @interface SupportedVersionTest { /** * Name of the counterpart test should be run on unsupported versions; such test must be * annoted with {@link UnsupportedVersionTest}, whith its {@code supportedVersionTest} * value point to the test being annotated with this annotation. */ String unsupportedVersionTest(); } /*** * Tells the rule to ignore an invalid API passed to {@link ApiTest}. * *

Should be used in cases where the API is being indirectly tested (for example, through a * shell command) and hence is not available in the test's classpath. * *

Should be used in conjunction with {@link ApiRequirements}. * */ @Retention(RUNTIME) @Target({TYPE, METHOD}) public @interface IgnoreInvalidApi { /** * Reason why the invalid API should be ignored. */ String reason(); } public static final class ExpectedVersionAssumptionViolationException extends AssumptionViolatedException { private static final long serialVersionUID = 1L; private final CarVersion mCarVersion; private final PlatformVersion mPlatformVersion; private final ApiRequirements mApiRequirements; ExpectedVersionAssumptionViolationException(Annotation annotation, CarVersion carVersion, PlatformVersion platformVersion, ApiRequirements apiRequirements) { super("Test annotated with @" + annotation.annotationType().getCanonicalName() + " when running on unsupported platform: CarVersion=" + carVersion + ", PlatformVersion=" + platformVersion + ", ApiRequirements=" + apiRequirements); mCarVersion = carVersion; mPlatformVersion = platformVersion; mApiRequirements = apiRequirements; } public CarVersion getCarVersion() { return mCarVersion; } public PlatformVersion getPlatformVersion() { return mPlatformVersion; } public ApiRequirements getApiRequirements() { return mApiRequirements; } } public static final class PlatformVersionMismatchExceptionNotThrownException extends IllegalStateException { private static final long serialVersionUID = 1L; private final CarVersion mCarVersion; private final PlatformVersion mPlatformVersion; private final ApiRequirements mApiRequirements; PlatformVersionMismatchExceptionNotThrownException(CarVersion carVersion, PlatformVersion platformVersion, ApiRequirements apiRequirements) { super("Test should throw " + PlatformVersionMismatchException.class.getSimpleName() + " when running on unsupported platform: CarVersion=" + carVersion + ", PlatformVersion=" + platformVersion + ", ApiRequirements=" + apiRequirements); mCarVersion = carVersion; mPlatformVersion = platformVersion; mApiRequirements = apiRequirements; } public CarVersion getCarVersion() { return mCarVersion; } public PlatformVersion getPlatformVersion() { return mPlatformVersion; } public ApiRequirements getApiRequirements() { return mApiRequirements; } } public static final class IncompatibleApiRequirementsException extends IllegalArgumentException { private static final long serialVersionUID = 1L; private final List mApis; private final List mApiRequirements; IncompatibleApiRequirementsException(List apis, List apiRequirements) { super("Incompatible API requirements (apis=" + apis + ", apiRequirements=" + apiRequirements + ") on test, consider splitting it into multiple methods"); mApis = apis; mApiRequirements = apiRequirements; } public List getApis() { return mApis; } public List getApiRequirements() { return mApiRequirements; } } }