1 /*
2  * Copyright (C) 2008 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 import com.android.tradefed.util.AbiUtils;
18 
19 import org.junit.runner.RunWith;
20 import org.w3c.dom.Document;
21 import org.w3c.dom.Element;
22 import org.w3c.dom.Node;
23 import org.w3c.dom.NodeList;
24 
25 import vogar.Expectation;
26 import vogar.ExpectationStore;
27 
28 import java.io.BufferedReader;
29 import java.io.File;
30 import java.io.FileInputStream;
31 import java.io.FileReader;
32 import java.io.IOException;
33 import java.lang.annotation.Annotation;
34 import java.lang.reflect.Method;
35 import java.lang.reflect.Modifier;
36 import java.util.ArrayList;
37 import java.util.Enumeration;
38 import java.util.HashSet;
39 import java.util.Iterator;
40 import java.util.LinkedHashMap;
41 import java.util.Map;
42 import java.util.Set;
43 import java.util.jar.JarEntry;
44 import java.util.jar.JarFile;
45 
46 import javax.xml.parsers.DocumentBuilderFactory;
47 import javax.xml.parsers.ParserConfigurationException;
48 
49 import junit.framework.TestCase;
50 
51 public class CollectAllTests extends DescriptionGenerator {
52 
53     private static final String ATTRIBUTE_RUNNER = "runner";
54     private static final String ATTRIBUTE_PACKAGE = "appPackageName";
55     private static final String ATTRIBUTE_NS = "appNameSpace";
56     private static final String ATTRIBUTE_TARGET = "targetNameSpace";
57     private static final String ATTRIBUTE_TARGET_BINARY = "targetBinaryName";
58     private static final String ATTRIBUTE_HOST_SIDE_ONLY = "hostSideOnly";
59     private static final String ATTRIBUTE_VM_HOST_TEST = "vmHostTest";
60     private static final String ATTRIBUTE_JAR_PATH = "jarPath";
61     private static final String ATTRIBUTE_JAVA_PACKAGE_FILTER = "javaPackageFilter";
62 
63     private static final String JAR_PATH = "LOCAL_JAR_PATH :=";
64     private static final String TEST_TYPE = "LOCAL_TEST_TYPE :";
65 
main(String[] args)66     public static void main(String[] args) {
67         if (args.length < 5 || args.length > 7) {
68             System.err.println("usage: CollectAllTests <output-file> <manifest-file> <jar-file> "
69                                + "<java-package> <architecture> [expectation-dir [makefile-file]]");
70             if (args.length != 0) {
71                 System.err.println("received:");
72                 for (String arg : args) {
73                     System.err.println("  " + arg);
74                 }
75             }
76             System.exit(1);
77         }
78 
79         final String outputPathPrefix = args[0];
80         File manifestFile = new File(args[1]);
81         String jarFileName = args[2];
82         final String javaPackageFilterArg = args[3] != null ? args[3].replaceAll("\\s+", "") : "";
83         final String[] javaPackagePrefixes;
84         // Validate the javaPackageFilter value if non-empty.
85         if (!javaPackageFilterArg.isEmpty()) {
86             javaPackagePrefixes = javaPackageFilterArg.split(":");
87             for (int i = 0; i < javaPackagePrefixes.length; ++i) {
88                 final String javaPackageFilter = javaPackagePrefixes[i];
89                 if (!isValidJavaPackage(javaPackageFilter)) {
90                     System.err.println("Invalid " + ATTRIBUTE_JAVA_PACKAGE_FILTER + ": " +
91                            javaPackageFilter);
92                     System.exit(1);
93                     return;
94                 } else {
95                     javaPackagePrefixes[i] = javaPackageFilter.trim() + ".";
96                 }
97             }
98         } else {
99             javaPackagePrefixes = new String[0];
100         }
101 
102         String architecture = args[4];
103         if (architecture == null || architecture.equals("")) {
104             System.err.println("Invalid architecture");
105             System.exit(1);
106             return;
107         }
108         String libcoreExpectationDir = (args.length > 5) ? args[5] : null;
109         String androidMakeFile = (args.length > 6) ? args[6] : null;
110 
111         final TestType testType = TestType.getTestType(androidMakeFile);
112 
113         Document manifest;
114         try {
115             manifest = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(
116                   new FileInputStream(manifestFile));
117         } catch (Exception e) {
118             System.err.println("cannot open manifest " + manifestFile);
119             e.printStackTrace();
120             System.exit(1);
121             return;
122         }
123 
124         Element documentElement = manifest.getDocumentElement();
125         documentElement.getAttribute("package");
126         final String runner = getElementAttribute(documentElement,
127                                                   "instrumentation",
128                                                   "android:name");
129         final String packageName = documentElement.getAttribute("package");
130         final String target = getElementAttribute(documentElement,
131                                                   "instrumentation",
132                                                   "android:targetPackage");
133 
134         String outputXmlFile = outputPathPrefix + ".xml";
135         final String xmlName = new File(outputPathPrefix).getName();
136         XMLGenerator xmlGenerator;
137         try {
138             xmlGenerator = new XMLGenerator(outputXmlFile) {
139                 {
140                     Node testPackageElem = mDoc.getDocumentElement();
141 
142                     setAttribute(testPackageElem, ATTRIBUTE_NAME, xmlName);
143                     setAttribute(testPackageElem, ATTRIBUTE_RUNNER, runner);
144                     setAttribute(testPackageElem, ATTRIBUTE_PACKAGE, packageName);
145                     setAttribute(testPackageElem, ATTRIBUTE_NS, packageName);
146                     setAttribute(testPackageElem, ATTRIBUTE_JAVA_PACKAGE_FILTER, javaPackageFilterArg);
147 
148                     if (testType.type == TestType.HOST_SIDE_ONLY) {
149                         setAttribute(testPackageElem, ATTRIBUTE_HOST_SIDE_ONLY, "true");
150                         setAttribute(testPackageElem, ATTRIBUTE_JAR_PATH, testType.jarPath);
151                     }
152 
153                     if (testType.type == TestType.VM_HOST_TEST) {
154                         setAttribute(testPackageElem, ATTRIBUTE_VM_HOST_TEST, "true");
155                         setAttribute(testPackageElem, ATTRIBUTE_JAR_PATH, testType.jarPath);
156                     }
157 
158                     if (!packageName.equals(target)) {
159                         setAttribute(testPackageElem, ATTRIBUTE_TARGET, target);
160                         setAttribute(testPackageElem, ATTRIBUTE_TARGET_BINARY, target);
161                     }
162                 }
163             };
164         } catch (ParserConfigurationException e) {
165             System.err.println("Can't initialize XML Generator " + outputXmlFile);
166             System.exit(1);
167             return;
168         }
169 
170         ExpectationStore libcoreVogarExpectationStore;
171         ExpectationStore ctsVogarExpectationStore;
172 
173         try {
174             libcoreVogarExpectationStore
175                     = VogarUtils.provideExpectationStore(libcoreExpectationDir);
176             ctsVogarExpectationStore = VogarUtils.provideExpectationStore(CTS_EXPECTATION_DIR);
177         } catch (IOException e) {
178             System.err.println("Can't initialize vogar expectation store from "
179                                + libcoreExpectationDir);
180             e.printStackTrace(System.err);
181             System.exit(1);
182             return;
183         }
184         ExpectationStore[] expectations = new ExpectationStore[] {
185             libcoreVogarExpectationStore, ctsVogarExpectationStore
186         };
187 
188         JarFile jarFile = null;
189         try {
190             jarFile = new JarFile(jarFileName);
191         } catch (Exception e) {
192             System.err.println("cannot open jarfile " + jarFileName);
193             e.printStackTrace();
194             System.exit(1);
195         }
196 
197         Map<String,TestClass> testCases = new LinkedHashMap<String, TestClass>();
198 
199         Enumeration<JarEntry> jarEntries = jarFile.entries();
200         while (jarEntries.hasMoreElements()) {
201             JarEntry jarEntry = jarEntries.nextElement();
202             String name = jarEntry.getName();
203             if (!name.endsWith(".class")) {
204                 continue;
205             }
206             String className
207                     = name.substring(0, name.length() - ".class".length()).replace('/', '.');
208 
209             boolean matchesPrefix = false;
210             if (javaPackagePrefixes.length > 0) {
211                 for (String javaPackagePrefix : javaPackagePrefixes) {
212                     if (className.startsWith(javaPackagePrefix)) {
213                         matchesPrefix = true;
214                     }
215                 }
216             } else {
217                 matchesPrefix = true;
218             }
219 
220             if (!matchesPrefix) {
221                 continue;
222             }
223 
224             // Avoid inner classes: they should not have tests and often they can have dependencies
225             // on test frameworks that need to be resolved and would need to be on the classpath.
226             // e.g. Mockito.
227             if (className.contains("$")) {
228                 continue;
229             }
230 
231             try {
232                 Class<?> klass = Class.forName(className,
233                                                false,
234                                                CollectAllTests.class.getClassLoader());
235                 final int modifiers = klass.getModifiers();
236                 if (Modifier.isAbstract(modifiers) || !Modifier.isPublic(modifiers)) {
237                     continue;
238                 }
239 
240                 final boolean isJunit4Class = isJunit4Class(klass);
241                 if (!isJunit4Class && !isJunit3Test(klass)) {
242                     continue;
243                 }
244 
245                 try {
246                     klass.getConstructor(new Class<?>[] { String.class } );
247                     addToTests(expectations, architecture, testCases, klass);
248                     continue;
249                 } catch (NoSuchMethodException e) {
250                 } catch (SecurityException e) {
251                     System.out.println("Known bug (Working as intended): problem with class "
252                             + className);
253                     e.printStackTrace();
254                 }
255 
256                 try {
257                     klass.getConstructor(new Class<?>[0]);
258                     addToTests(expectations, architecture, testCases, klass);
259                     continue;
260                 } catch (NoSuchMethodException e) {
261                 } catch (SecurityException e) {
262                     System.out.println("Known bug (Working as intended): problem with class "
263                             + className);
264                     e.printStackTrace();
265                 }
266             } catch (ClassNotFoundException e) {
267                 System.out.println("class not found " + className);
268                 e.printStackTrace();
269                 System.exit(1);
270             }
271         }
272 
273         for (Iterator<TestClass> iterator = testCases.values().iterator(); iterator.hasNext();) {
274             TestClass type = iterator.next();
275             xmlGenerator.addTestClass(type);
276         }
277 
278         try {
279             xmlGenerator.dump();
280         } catch (Exception e) {
281             System.err.println("cannot dump xml to " + outputXmlFile);
282             e.printStackTrace();
283             System.exit(1);
284         }
285     }
286 
287     private static class TestType {
288         private static final int HOST_SIDE_ONLY = 1;
289         private static final int DEVICE_SIDE_ONLY = 2;
290         private static final int VM_HOST_TEST = 3;
291 
292         private final int type;
293         private final String jarPath;
294 
TestType(int type, String jarPath)295         private TestType (int type, String jarPath) {
296             this.type = type;
297             this.jarPath = jarPath;
298         }
299 
getTestType(String makeFileName)300         private static TestType getTestType(String makeFileName) {
301             if (makeFileName == null || makeFileName.isEmpty()) {
302                 return new TestType(DEVICE_SIDE_ONLY, null);
303             }
304             int type = TestType.DEVICE_SIDE_ONLY;
305             String jarPath = null;
306             try {
307                 BufferedReader reader = new BufferedReader(new FileReader(makeFileName));
308                 String line;
309 
310                 while ((line = reader.readLine())!=null) {
311                     if (line.startsWith(TEST_TYPE)) {
312                         if (line.indexOf(ATTRIBUTE_VM_HOST_TEST) >= 0) {
313                             type = VM_HOST_TEST;
314                         } else {
315                             type = HOST_SIDE_ONLY;
316                         }
317                     } else if (line.startsWith(JAR_PATH)) {
318                         jarPath = line.substring(JAR_PATH.length(), line.length()).trim();
319                     }
320                 }
321                 reader.close();
322             } catch (IOException e) {
323             }
324             return new TestType(type, jarPath);
325         }
326     }
327 
getElement(Element element, String tagName)328     private static Element getElement(Element element, String tagName) {
329         NodeList elements = element.getElementsByTagName(tagName);
330         if (elements.getLength() > 0) {
331             return (Element) elements.item(0);
332         } else {
333             return null;
334         }
335     }
336 
getElementAttribute(Element element, String elementName, String attributeName)337     private static String getElementAttribute(Element element,
338                                               String elementName,
339                                               String attributeName) {
340         Element e = getElement(element, elementName);
341         if (e != null) {
342             return e.getAttribute(attributeName);
343         } else {
344             return "";
345         }
346     }
347 
getKnownFailure(final Class<?> testClass, final String testName)348     private static String getKnownFailure(final Class<?> testClass,
349             final String testName) {
350         return getAnnotation(testClass, testName, KNOWN_FAILURE);
351     }
352 
isKnownFailure(final Class<?> testClass, final String testName)353     private static boolean isKnownFailure(final Class<?> testClass,
354             final String testName) {
355         return getAnnotation(testClass, testName, KNOWN_FAILURE) != null;
356     }
357 
isSuppressed(final Class<?> testClass, final String testName)358     private static boolean isSuppressed(final Class<?> testClass,
359             final String testName)  {
360         return getAnnotation(testClass, testName, SUPPRESSED_TEST) != null;
361     }
362 
getAnnotation(final Class<?> testClass, final String testName, final String annotationName)363     private static String getAnnotation(final Class<?> testClass,
364             final String testName, final String annotationName) {
365         try {
366             Method testMethod = testClass.getMethod(testName, (Class[])null);
367             Annotation[] annotations = testMethod.getAnnotations();
368             for (Annotation annot : annotations) {
369 
370                 if (annot.annotationType().getName().equals(annotationName)) {
371                     String annotStr = annot.toString();
372                     String knownFailure = null;
373                     if (annotStr.contains("(value=")) {
374                         knownFailure =
375                             annotStr.substring(annotStr.indexOf("=") + 1,
376                                     annotStr.length() - 1);
377 
378                     }
379 
380                     if (knownFailure == null) {
381                         knownFailure = "true";
382                     }
383 
384                     return knownFailure;
385                 }
386 
387             }
388 
389         } catch (NoSuchMethodException e) {
390         }
391 
392         return null;
393     }
394 
addToTests(ExpectationStore[] expectations, String architecture, Map<String,TestClass> testCases, Class<?> testClass)395     private static void addToTests(ExpectationStore[] expectations,
396                                    String architecture,
397                                    Map<String,TestClass> testCases,
398                                    Class<?> testClass) {
399         Set<String> testNames = new HashSet<String>();
400 
401         boolean isJunit3Test = isJunit3Test(testClass);
402 
403         Method[] testMethods = testClass.getMethods();
404         for (Method testMethod : testMethods) {
405             String testName = testMethod.getName();
406             if (testNames.contains(testName)) {
407                 continue;
408             }
409 
410             /* Make sure the method has the right signature. */
411             if (!Modifier.isPublic(testMethod.getModifiers())) {
412                 continue;
413             }
414             if (!testMethod.getReturnType().equals(Void.TYPE)) {
415                 continue;
416             }
417             if (testMethod.getParameterTypes().length != 0) {
418                 continue;
419             }
420 
421             if ((isJunit3Test && !testName.startsWith("test"))
422                     || (!isJunit3Test && !isJunit4TestMethod(testMethod))) {
423                 continue;
424             }
425 
426             testNames.add(testName);
427             addToTests(expectations, architecture, testCases, testClass, testName);
428         }
429     }
430 
addToTests(ExpectationStore[] expectations, String architecture, Map<String,TestClass> testCases, Class<?> test, String testName)431     private static void addToTests(ExpectationStore[] expectations,
432                                    String architecture,
433                                    Map<String,TestClass> testCases,
434                                    Class<?> test,
435                                    String testName) {
436 
437         String testClassName = test.getName();
438         String knownFailure = getKnownFailure(test, testName);
439 
440         if (isKnownFailure(test, testName)) {
441             System.out.println("ignoring known failure: " + test + "#" + testName);
442             return;
443         } else if (isSuppressed(test, testName)) {
444             System.out.println("ignoring suppressed test: " + test + "#" + testName);
445             return;
446         } else if (VogarUtils.isVogarKnownFailure(expectations,
447                                                   testClassName,
448                                                   testName)) {
449             System.out.println("ignoring expectation known failure: " + test
450                                + "#" + testName);
451             return;
452         }
453 
454         Set<String> supportedAbis = VogarUtils.extractSupportedAbis(architecture,
455                                                                     expectations,
456                                                                     testClassName,
457                                                                     testName);
458         int timeoutInMinutes = VogarUtils.timeoutInMinutes(expectations,
459                                                            testClassName,
460                                                            testName);
461         TestClass testClass;
462         if (testCases.containsKey(testClassName)) {
463             testClass = testCases.get(testClassName);
464         } else {
465             testClass = new TestClass(testClassName, new ArrayList<TestMethod>());
466             testCases.put(testClassName, testClass);
467         }
468 
469         testClass.mCases.add(new TestMethod(testName, "", "", supportedAbis,
470               knownFailure, false, false, timeoutInMinutes));
471     }
472 
isJunit3Test(Class<?> klass)473     private static boolean isJunit3Test(Class<?> klass) {
474         return TestCase.class.isAssignableFrom(klass);
475     }
476 
isJunit4Class(Class<?> klass)477     private static boolean isJunit4Class(Class<?> klass) {
478         for (Annotation a : klass.getAnnotations()) {
479             if (RunWith.class.isAssignableFrom(a.annotationType())) {
480                 // @RunWith is currently not supported for CTS tests because tradefed cannot handle
481                 // a single test spawning other tests with different names.
482                 System.out.println("Skipping test class " + klass.getName()
483                         + ": JUnit4 @RunWith is not supported");
484                 return false;
485             }
486         }
487 
488         for (Method m : klass.getMethods()) {
489             if (isJunit4TestMethod(m)) {
490                 return true;
491             }
492         }
493 
494         return false;
495     }
496 
isJunit4TestMethod(Method method)497     private static boolean isJunit4TestMethod(Method method) {
498         for (Annotation a : method.getAnnotations()) {
499             if (org.junit.Test.class.isAssignableFrom(a.annotationType())) {
500                 return true;
501             }
502         }
503 
504         return false;
505     }
506 
507     /**
508      * Determines if a given string is a valid java package name
509      * @param javaPackageName
510      * @return true if it is valid, false otherwise
511      */
isValidJavaPackage(String javaPackageName)512     private static boolean isValidJavaPackage(String javaPackageName) {
513         String[] strSections = javaPackageName.split(".");
514         for (String strSection : strSections) {
515           if (!isValidJavaIdentifier(strSection)) {
516               return false;
517           }
518         }
519         return true;
520     }
521 
522     /**
523      * Determines if a given string is a valid java identifier.
524      * @param javaIdentifier
525      * @return true if it is a valid identifier, false otherwise
526      */
isValidJavaIdentifier(String javaIdentifier)527     private static boolean isValidJavaIdentifier(String javaIdentifier) {
528         if (javaIdentifier.length() == 0 ||
529                 !Character.isJavaIdentifierStart(javaIdentifier.charAt(0))) {
530             return false;
531         }
532         for (int i = 1; i < javaIdentifier.length(); i++) {
533             if (!Character.isJavaIdentifierPart(javaIdentifier.charAt(i))) {
534                 return false;
535             }
536         }
537         return true;
538     }
539 }
540