1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.cts.apicoverage;
18 
19 import com.android.cts.apicoverage.TestSuiteProto.*;
20 import com.android.cts.apicoverage.CtsReportProto.*;
21 
22 import org.xml.sax.SAXException;
23 
24 import java.io.FileInputStream;
25 import java.io.FileWriter;
26 import java.io.IOException;
27 import java.io.PrintWriter;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 
32 /**
33  * Class that outputs an XML report of the {@link ApiCoverage} collected. It can be viewed in a
34  * browser when used with the api-coverage.css and api-coverage.xsl files.
35  */
36 class ComparisonReport {
37     public static final String TEST_RESULT_FILE_NAME = "test_result.xml";
38     // [module].[class]#[method]
39     public static final String TESTCASE_NAME_FORMAT = "%s.%s#%s";
40 
printUsage()41     private static void printUsage() {
42         System.out.println("Usage: ReportChecker [OPTION]...");
43         System.out.println();
44         System.out.println("Generates a comparsion report about Test Suite Content and Report.");
45         System.out.println();
46         System.out.println("Options:");
47         System.out.println("  -o FILE                output file of missing test case list");
48         System.out.println("  -e FILE                output file of extra test case list ");
49         System.out.println("  -r FILE                test report PB file");
50         System.out.println("  -t FILE                test suite content PB file");
51 
52         System.out.println();
53         System.exit(1);
54     }
55 
isKnownFailure(String tsName, List<String> knownFailures)56     private static boolean isKnownFailure(String tsName, List<String> knownFailures) {
57         for (String kf : knownFailures) {
58             if (tsName.startsWith(kf)) {
59                 return true;
60             }
61         }
62         return false;
63     }
64 
writeComparsionReport( TestSuite ts, CtsReport tsReport, String outputFileName, String outputExtraTestFileName, List<String> knownFailures, String extraTestSubPlanFileName)65     private static void writeComparsionReport(
66             TestSuite ts,
67             CtsReport tsReport,
68             String outputFileName,
69             String outputExtraTestFileName,
70             List<String> knownFailures,
71             String extraTestSubPlanFileName)
72             throws IOException {
73 
74         HashMap<String, String> testCaseMap = new HashMap<String, String>();
75         HashMap<String, String> extraTestCaseMap = new HashMap<String, String>();
76         // Add all test case into a HashMap
77         for (TestSuite.Package pkg : ts.getPackagesList()) {
78             for (TestSuite.Package.Class cls : pkg.getClassesList()) {
79                 for (TestSuite.Package.Class.Method mtd : cls.getMethodsList()) {
80                     String testCaseName =
81                             String.format(
82                                     TESTCASE_NAME_FORMAT,
83                                     pkg.getName(),
84                                     cls.getName(),
85                                     mtd.getName());
86                     // Filter out known failures
87                     if (!isKnownFailure(testCaseName, knownFailures)) {
88                         String testApkType = String.format("%s,%s", cls.getApk(), pkg.getType());
89                         testCaseMap.put(testCaseName, testApkType);
90                         extraTestCaseMap.put(testCaseName, testApkType);
91                         System.out.printf("add: %s: %s\n", testCaseName, testApkType);
92                     } else {
93                         System.out.printf("KF: %s\n", testCaseName);
94                     }
95                 }
96             }
97         }
98         System.out.printf(
99                 "Before: testCaseMap entries: %d & extraTestCaseMap: %d\n",
100                 testCaseMap.size(), extraTestCaseMap.size());
101 
102         //List missing Test Cases
103         FileWriter fWriter = new FileWriter(outputFileName);
104         PrintWriter pWriter = new PrintWriter(fWriter);
105 
106         pWriter.printf("no,Module,TestCase,abi\n");
107         int i = 1;
108         for (CtsReport.TestPackage tPkg : tsReport.getTestPackageList()) {
109             for (CtsReport.TestPackage.TestSuite module : tPkg.getTestSuiteList()) {
110                 for (CtsReport.TestPackage.TestSuite.TestCase testCase : module.getTestCaseList()) {
111                     for (CtsReport.TestPackage.TestSuite.TestCase.Test test :
112                             testCase.getTestList()) {
113                         String testCaseName =
114                                 String.format(
115                                         TESTCASE_NAME_FORMAT,
116                                         tPkg.getName(),
117                                         testCase.getName(),
118                                         test.getName());
119 
120                         System.out.printf(
121                                 "rm: %s, %s\n",
122                                 testCaseName, extraTestCaseMap.remove(testCaseName));
123                         if (!testCaseMap.containsKey(testCaseName)) {
124                             pWriter.printf(
125                                     "%d,%s,%s,%s\n",
126                                     i++, module.getName(), testCaseName, tPkg.getAbi());
127                         }
128                     }
129                 }
130             }
131         }
132         pWriter.close();
133 
134         //List extra Test Cases
135         fWriter = new FileWriter(outputExtraTestFileName);
136         pWriter = new PrintWriter(fWriter);
137         pWriter.printf("no,TestCase,Apk,Type\n");
138         int j = 1;
139         for (Map.Entry<String, String> test : extraTestCaseMap.entrySet()) {
140             pWriter.printf("%d,%s,%s\n", j++, test.getKey(), test.getValue());
141         }
142         pWriter.close();
143         System.out.printf(
144                 "After: testCaseMap entries: %d & extraTestCaseMap: %d\n",
145                 testCaseMap.size(), extraTestCaseMap.size());
146 
147         if (null != extraTestSubPlanFileName) {
148             //Create extra Test Cases subplan
149             fWriter = new FileWriter(extraTestSubPlanFileName);
150             pWriter = new PrintWriter(fWriter);
151             pWriter.println("<?xml version=\'1.0\' encoding=\'UTF-8\' standalone=\'no\' ?>");
152             pWriter.println("<SubPlan version=\'2.0\'>");
153             for (Map.Entry<String, String> test : extraTestCaseMap.entrySet()) {
154                 pWriter.printf(
155                         "  <Entry include=\"%s\"/>\n", test.getKey().replaceFirst("\\.", " "));
156             }
157             pWriter.println("</SubPlan>");
158             pWriter.close();
159         }
160 
161     }
162 
main(String[] args)163     public static void main(String[] args) throws IOException, SAXException, Exception {
164         String outputFileName = "./missingTestCase.csv";
165         String testReportFileName = "./extraTestCase.csv";
166         String testSuiteFileName = null;
167         String testSuiteContentFileName = null;
168         String outputExtraTestFileName = null;
169         String extraTestSubPlanFileName = null;
170         int numTestModule = 0;
171 
172         for (int i = 0; i < args.length; i++) {
173             if (args[i].startsWith("-")) {
174                 if ("-o".equals(args[i])) {
175                     outputFileName = getExpectedArg(args, ++i);
176                 } else if ("-r".equals(args[i])) {
177                     testReportFileName = getExpectedArg(args, ++i);
178                 } else if ("-e".equals(args[i])) {
179                     outputExtraTestFileName = getExpectedArg(args, ++i);
180                 } else if ("-t".equals(args[i])) {
181                     testSuiteFileName = getExpectedArg(args, ++i);
182                 } else if ("-s".equals(args[i])) {
183                     testSuiteContentFileName = getExpectedArg(args, ++i);
184                 } else if ("-p".equals(args[i])) {
185                     extraTestSubPlanFileName = getExpectedArg(args, ++i);
186                 } else {
187                     printUsage();
188                 }
189             } else {
190                 printUsage();
191             }
192         }
193 
194         if (null == testReportFileName || null == testSuiteContentFileName) {
195             printUsage();
196         }
197 
198         TestSuiteContent tsContent =
199                 TestSuiteContent.parseFrom(new FileInputStream(testSuiteContentFileName));
200         TestSuite ts = TestSuite.parseFrom(new FileInputStream(testSuiteFileName));
201         CtsReport tsReport = CtsReport.parseFrom(new FileInputStream(testReportFileName));
202 
203         writeComparsionReport(
204                 ts,
205                 tsReport,
206                 outputFileName,
207                 outputExtraTestFileName,
208                 tsContent.getKnownFailuresList(),
209                 extraTestSubPlanFileName);
210         System.err.printf("%s", tsContent.getKnownFailuresList());
211     }
212 
213     /** Get the argument or print out the usage and exit. */
getExpectedArg(String[] args, int index)214     private static String getExpectedArg(String[] args, int index) {
215         if (index < args.length) {
216             return args[index];
217         } else {
218             printUsage();
219             return null; // Never will happen because printUsage will call exit(1)
220         }
221     }
222 }
223