1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.common.tradefed.result; 18 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 20 import com.android.compatibility.common.tradefed.testtype.ISubPlan; 21 import com.android.compatibility.common.tradefed.testtype.SubPlan; 22 import com.android.compatibility.common.util.ICaseResult; 23 import com.android.compatibility.common.util.IInvocationResult; 24 import com.android.compatibility.common.util.IModuleResult; 25 import com.android.compatibility.common.util.ITestResult; 26 import com.android.compatibility.common.util.InvocationResult; 27 import com.android.compatibility.common.util.ResultHandler; 28 import com.android.compatibility.common.util.TestFilter; 29 import com.android.compatibility.common.util.TestStatus; 30 import com.android.tradefed.build.BuildInfo; 31 import com.android.tradefed.build.IBuildInfo; 32 import com.android.tradefed.config.ArgsOptionParser; 33 import com.android.tradefed.util.AbiUtils; 34 import com.android.tradefed.util.FileUtil; 35 36 import junit.framework.TestCase; 37 38 import java.io.File; 39 import java.io.FileNotFoundException; 40 import java.util.Arrays; 41 import java.util.Set; 42 43 public class SubPlanHelperTest extends TestCase { 44 45 // Values used to populate mock results 46 private static final String SUITE_NAME = "CTS"; 47 private static final String SUITE_VERSION = "5.0"; 48 private static final String SUITE_PLAN = "cts"; 49 private static final String SUITE_BUILD = "12345"; 50 private static final String NAME_A = "ModuleA"; 51 private static final String NAME_B = "ModuleB"; 52 private static final String ABI = "mips64"; 53 private static final String ID_A = AbiUtils.createId(ABI, NAME_A); 54 private static final String ID_B = AbiUtils.createId(ABI, NAME_B); 55 private static final String BUILD_ID = "build_id"; 56 private static final String BUILD_PRODUCT = "build_product"; 57 private static final String EXAMPLE_BUILD_ID = "XYZ"; 58 private static final String EXAMPLE_BUILD_PRODUCT = "wolverine"; 59 private static final String DEVICE_A = "device123"; 60 private static final String DEVICE_B = "device456"; 61 private static final String CLASS_A = "android.test.Foor"; 62 private static final String CLASS_B = "android.test.Bar"; 63 private static final String METHOD_1 = "testBlah1"; 64 private static final String METHOD_2 = "testBlah2"; 65 private static final String METHOD_3 = "testBlah3"; 66 private static final String METHOD_4 = "testBlah4"; 67 private static final long START_MS = 1431586801000L; 68 private static final long END_MS = 1431673199000L; 69 private static final String REFERENCE_URL="http://android.com"; 70 private static final String LOG_URL ="file:///path/to/logs"; 71 private static final String COMMAND_LINE_ARGS = "cts -m CtsMyModuleTestCases"; 72 73 private static final String SP_NAME = "testsubplan"; 74 private static final String SP_SESSION = "0"; 75 private static final String SP_RESULT_TYPE_FAILED = "failed"; 76 private static final String SP_RESULT_TYPE_NOT_EXECUTED = "not_executed"; 77 78 private static final String PARAM_SUFFIX = "[0]"; 79 80 private CompatibilityBuildHelper mBuildHelper; 81 private SubPlanHelper mSubPlanHelper; 82 83 private File mResultsDir = null; 84 private File mResultDir = null; 85 private File mSubPlansDir = null; 86 87 @Override setUp()88 public void setUp() throws Exception { 89 mResultsDir = FileUtil.createTempDir("results"); 90 mResultDir = FileUtil.createTempDir("12345", mResultsDir); 91 mSubPlansDir = FileUtil.createTempDir("subplans"); 92 mBuildHelper = new SpctMockCompatibilityBuildHelper(new BuildInfo("0", "")); 93 populateResults(); 94 95 mSubPlanHelper = new SubPlanHelper(); 96 ArgsOptionParser optionParser = new ArgsOptionParser(mSubPlanHelper); 97 optionParser.parse(Arrays.asList( 98 "-n", SP_NAME, 99 "--session", SP_SESSION, 100 "--result-type", SP_RESULT_TYPE_FAILED, 101 "--result-type", SP_RESULT_TYPE_NOT_EXECUTED)); 102 } 103 104 @Override tearDown()105 public void tearDown() throws Exception { 106 if (mResultsDir != null) { 107 FileUtil.recursiveDelete(mResultsDir); 108 } 109 if (mSubPlansDir != null) { 110 FileUtil.recursiveDelete(mSubPlansDir); 111 } 112 } 113 testCreateSubPlan()114 public void testCreateSubPlan() throws Exception { 115 ISubPlan plan = mSubPlanHelper.createSubPlan(mBuildHelper); 116 Set<String> planIncludes = plan.getIncludeFilters(); 117 Set<String> planExcludes = plan.getExcludeFilters(); 118 TestFilter mf1 = new TestFilter(ABI, NAME_A, null); 119 TestFilter tf1 = new TestFilter(ABI, NAME_A, String.format("%s#%s", CLASS_A, METHOD_1)); 120 TestFilter tf3 = new TestFilter(ABI, NAME_B, String.format("%s#%s", CLASS_B, METHOD_3)); 121 assertTrue(planIncludes.contains("CtsMyModuleTestCases")); // command-line '-m' arg 122 assertTrue(planIncludes.contains(mf1.toString())); // include module with not-executed test 123 assertTrue(planExcludes.contains(tf1.toString())); // exclude passing test in that module 124 assertTrue(planIncludes.contains(tf3.toString())); // include failure in executed module 125 } 126 testAddInclude()127 public void testAddInclude() throws Exception { 128 ISubPlan subPlan = new SubPlan(); 129 TestFilter tf = new TestFilter(ABI, NAME_A, String.format("%s#%s", CLASS_A, METHOD_1)); 130 SubPlanHelper.addIncludeToSubPlan(subPlan, tf); 131 Set<String> includes = subPlan.getIncludeFilters(); 132 assertTrue(includes.contains(tf.toString())); 133 } 134 testAddExclude()135 public void testAddExclude() throws Exception { 136 ISubPlan subPlan = new SubPlan(); 137 TestFilter tf = new TestFilter(ABI, NAME_A, String.format("%s#%s", CLASS_A, METHOD_1)); 138 SubPlanHelper.addExcludeToSubPlan(subPlan, tf); 139 Set<String> excludes = subPlan.getExcludeFilters(); 140 assertTrue(excludes.contains(tf.toString())); 141 } 142 testAddParameterizedInclude()143 public void testAddParameterizedInclude() throws Exception { 144 ISubPlan subPlan = new SubPlan(); 145 TestFilter filterWithSuffix = new TestFilter(ABI, NAME_A, 146 String.format("%s#%s%s", CLASS_A, METHOD_1, PARAM_SUFFIX)); 147 TestFilter filterWithoutSuffix = new TestFilter(ABI, NAME_A, 148 String.format("%s#%s", CLASS_A, METHOD_1)); 149 SubPlanHelper.addIncludeToSubPlan(subPlan, filterWithSuffix); 150 Set<String> includes = subPlan.getIncludeFilters(); 151 assertTrue(includes.contains(filterWithoutSuffix.toString())); 152 } 153 testAddParameterizedExclude()154 public void testAddParameterizedExclude() throws Exception { 155 ISubPlan subPlan = new SubPlan(); 156 TestFilter filterWithSuffix = new TestFilter(ABI, NAME_A, 157 String.format("%s#%s%s", CLASS_A, METHOD_1, PARAM_SUFFIX)); 158 SubPlanHelper.addExcludeToSubPlan(subPlan, filterWithSuffix); 159 Set<String> excludes = subPlan.getExcludeFilters(); 160 assertTrue(excludes.isEmpty()); 161 } 162 populateResults()163 private void populateResults() throws Exception { 164 // copied from ResultHandlerTest 165 IInvocationResult result = new InvocationResult(); 166 result.setStartTime(START_MS); 167 result.setTestPlan(SUITE_PLAN); 168 result.addDeviceSerial(DEVICE_A); 169 result.addDeviceSerial(DEVICE_B); 170 result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID); 171 result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT); 172 IModuleResult moduleA = result.getOrCreateModule(ID_A); 173 moduleA.setDone(false); 174 ICaseResult moduleACase = moduleA.getOrCreateResult(CLASS_A); 175 ITestResult moduleATest1 = moduleACase.getOrCreateResult(METHOD_1); 176 moduleATest1.setResultStatus(TestStatus.PASS); 177 ITestResult moduleATest2 = moduleACase.getOrCreateResult(METHOD_2); 178 moduleATest2.setResultStatus(null); // not executed test 179 180 IModuleResult moduleB = result.getOrCreateModule(ID_B); 181 moduleB.setDone(true); 182 ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B); 183 ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3); 184 moduleBTest3.setResultStatus(TestStatus.FAIL); 185 ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4); 186 moduleBTest4.setResultStatus(TestStatus.PASS); 187 188 // Serialize to file 189 ResultHandler.writeResults( 190 SUITE_NAME, 191 SUITE_VERSION, 192 SUITE_PLAN, 193 SUITE_BUILD, 194 result, 195 mResultDir, 196 START_MS, 197 END_MS, 198 REFERENCE_URL, 199 LOG_URL, 200 COMMAND_LINE_ARGS, 201 null); 202 } 203 204 private class SpctMockCompatibilityBuildHelper extends CompatibilityBuildHelper { 205 SpctMockCompatibilityBuildHelper(IBuildInfo buildInfo)206 public SpctMockCompatibilityBuildHelper(IBuildInfo buildInfo) { 207 super(buildInfo); 208 } 209 210 @Override getResultsDir()211 public File getResultsDir() throws FileNotFoundException { 212 return mResultsDir; 213 } 214 215 @Override getSubPlansDir()216 public File getSubPlansDir() throws FileNotFoundException { 217 return mSubPlansDir; 218 } 219 } 220 } 221