1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite;
17 
18 import com.android.tradefed.config.ConfigurationException;
19 import com.android.tradefed.config.IConfiguration;
20 import com.android.tradefed.config.IDeviceConfiguration;
21 import com.android.tradefed.config.OptionCopier;
22 import com.android.tradefed.targetprep.ITargetPreparer;
23 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer;
24 import com.android.tradefed.testtype.IAbiReceiver;
25 import com.android.tradefed.testtype.IRemoteTest;
26 import com.android.tradefed.testtype.IShardableTest;
27 
28 import java.util.ArrayList;
29 import java.util.Collection;
30 import java.util.LinkedHashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Map.Entry;
34 
35 /**
36  * Helper to split a list of modules represented by {@link IConfiguration} into a list of execution
37  * units represented by {@link ModuleDefinition}.
38  *
39  * <p>Each configuration may generate 1 or more {@link ModuleDefinition} depending on its options
40  * and test types:
41  *
42  * <ul>
43  *   <li>A non-shardable {@link IConfiguration} will generate a single {@link ModuleDefinition}.
44  *   <li>A shardable {@link IConfiguration} will generate a number of ModuleDefinition linked to the
45  *       {@link IRemoteTest} properties:
46  *       <ul>
47  *         <li>A non - {@link IShardableTest} will generate a single ModuleDefinition.
48  *         <li>A {@link IShardableTest} generates one ModuleDefinition per tests returned by {@link
49  *             IShardableTest#split()}.
50  *       </ul>
51  *
52  * </ul>
53  */
54 public class ModuleSplitter {
55 
56     /**
57      * Create a List of executable unit {@link ModuleDefinition}s based on the map of configuration
58      * that was loaded.
59      *
60      * @param runConfig {@link LinkedHashMap} loaded from {@link ITestSuite#loadTests()}.
61      * @param shardCount a shard count hint to help with sharding.
62      * @return List of {@link ModuleDefinition}
63      */
splitConfiguration( LinkedHashMap<String, IConfiguration> runConfig, int shardCount, boolean dynamicModule)64     public static List<ModuleDefinition> splitConfiguration(
65             LinkedHashMap<String, IConfiguration> runConfig,
66             int shardCount,
67             boolean dynamicModule) {
68         if (dynamicModule) {
69             // We maximize the sharding for dynamic to reduce time difference between first and
70             // last shard as much as possible. Overhead is low due to our test pooling.
71             shardCount *= 2;
72         }
73         List<ModuleDefinition> runModules = new ArrayList<>();
74         for (Entry<String, IConfiguration> configMap : runConfig.entrySet()) {
75             // Check that it's a valid configuration for suites, throw otherwise.
76             ValidateSuiteConfigHelper.validateConfig(configMap.getValue());
77 
78             createAndAddModule(
79                     runModules,
80                     configMap.getKey(),
81                     configMap.getValue(),
82                     shardCount,
83                     dynamicModule);
84         }
85         return runModules;
86     }
87 
createAndAddModule( List<ModuleDefinition> currentList, String moduleName, IConfiguration config, int shardCount, boolean dynamicModule)88     private static void createAndAddModule(
89             List<ModuleDefinition> currentList,
90             String moduleName,
91             IConfiguration config,
92             int shardCount,
93             boolean dynamicModule) {
94         // If this particular configuration module is declared as 'not shardable' we take it whole
95         // but still split the individual IRemoteTest in a pool.
96         if (config.getConfigurationDescription().isNotShardable()
97                 || (!dynamicModule
98                         && config.getConfigurationDescription().isNotStrictShardable())) {
99             for (int i = 0; i < config.getTests().size(); i++) {
100                 if (dynamicModule) {
101                     ModuleDefinition module =
102                             new ModuleDefinition(
103                                     moduleName,
104                                     config.getTests(),
105                                     clonePreparersMap(config),
106                                     clonePreparers(config.getMultiTargetPreparers()),
107                                     config);
108                     currentList.add(module);
109                 } else {
110                     addModuleToListFromSingleTest(
111                             currentList, config.getTests().get(i), moduleName, config);
112                 }
113             }
114             return;
115         }
116 
117         // If configuration is possibly shardable we attempt to shard it.
118         for (IRemoteTest test : config.getTests()) {
119             if (test instanceof IShardableTest) {
120                 Collection<IRemoteTest> shardedTests = ((IShardableTest) test).split(shardCount);
121                 if (shardedTests != null) {
122                     // Test did shard we put the shard pool in ModuleDefinition which has a polling
123                     // behavior on the pool.
124                     if (dynamicModule) {
125                         for (int i = 0; i < shardCount; i++) {
126                             ModuleDefinition module =
127                                     new ModuleDefinition(
128                                             moduleName,
129                                             shardedTests,
130                                             clonePreparersMap(config),
131                                             clonePreparers(config.getMultiTargetPreparers()),
132                                             config);
133                             currentList.add(module);
134                         }
135                     } else {
136                         // We create independent modules with each sharded test.
137                         for (IRemoteTest moduleTest : shardedTests) {
138                             addModuleToListFromSingleTest(
139                                     currentList, moduleTest, moduleName, config);
140                         }
141                     }
142                     continue;
143                 }
144             }
145             // test is not shardable or did not shard
146             addModuleToListFromSingleTest(currentList, test, moduleName, config);
147         }
148     }
149 
150     /**
151      * Helper to add a new {@link ModuleDefinition} to our list of Modules from a single {@link
152      * IRemoteTest}.
153      */
addModuleToListFromSingleTest( List<ModuleDefinition> currentList, IRemoteTest test, String moduleName, IConfiguration config)154     private static void addModuleToListFromSingleTest(
155             List<ModuleDefinition> currentList,
156             IRemoteTest test,
157             String moduleName,
158             IConfiguration config) {
159         List<IRemoteTest> testList = new ArrayList<>();
160         testList.add(test);
161         ModuleDefinition module =
162                 new ModuleDefinition(
163                         moduleName,
164                         testList,
165                         clonePreparersMap(config),
166                         clonePreparers(config.getMultiTargetPreparers()),
167                         config);
168         currentList.add(module);
169     }
170 
171     /**
172      * Deep clone a list of {@link ITargetPreparer} or {@link IMultiTargetPreparer}. We are ensured
173      * to find a default constructor with no arguments since that's the expectation from Tradefed
174      * when loading configuration. Cloning preparers is required since they may be stateful and we
175      * cannot share instance across devices.
176      */
clonePreparers(List<T> preparerList)177     private static <T> List<T> clonePreparers(List<T> preparerList) {
178         List<T> clones = new ArrayList<>();
179         for (T prep : preparerList) {
180             try {
181                 @SuppressWarnings("unchecked")
182                 T clone = (T) prep.getClass().newInstance();
183                 OptionCopier.copyOptions(prep, clone);
184                 // Ensure we copy the Abi too.
185                 if (clone instanceof IAbiReceiver) {
186                     ((IAbiReceiver) clone).setAbi(((IAbiReceiver) prep).getAbi());
187                 }
188                 clones.add(clone);
189             } catch (InstantiationException | IllegalAccessException | ConfigurationException e) {
190                 throw new RuntimeException(e);
191             }
192         }
193         return clones;
194     }
195 
196     /** Deep cloning of potentially multi-device preparers. */
clonePreparersMap(IConfiguration config)197     private static Map<String, List<ITargetPreparer>> clonePreparersMap(IConfiguration config) {
198         Map<String, List<ITargetPreparer>> res = new LinkedHashMap<>();
199         for (IDeviceConfiguration holder : config.getDeviceConfig()) {
200             List<ITargetPreparer> preparers = new ArrayList<>();
201             res.put(holder.getDeviceName(), preparers);
202             preparers.addAll(clonePreparers(holder.getTargetPreparers()));
203         }
204         return res;
205     }
206 }
207