1 /*
2  * Copyright (c) 2017 Google Inc. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you
5  * may not use this file except in compliance with the License. You may
6  * obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13  * implied. See the License for the specific language governing
14  * permissions and limitations under the License.
15  */
16 
17 package com.android.vts.servlet;
18 
19 import com.android.vts.entity.CodeCoverageEntity;
20 import com.android.vts.entity.DeviceInfoEntity;
21 import com.android.vts.entity.TestCoverageStatusEntity;
22 import com.android.vts.entity.TestEntity;
23 import com.android.vts.entity.TestRunEntity;
24 
25 import com.android.vts.proto.VtsReportMessage;
26 import com.android.vts.util.FilterUtil;
27 import com.google.cloud.datastore.Datastore;
28 import com.google.cloud.datastore.DatastoreOptions;
29 import com.google.cloud.datastore.PathElement;
30 import com.google.cloud.datastore.StructuredQuery.CompositeFilter;
31 import com.google.cloud.datastore.StructuredQuery.Filter;
32 import com.google.cloud.datastore.StructuredQuery.PropertyFilter;
33 import com.google.gson.Gson;
34 import com.google.visualization.datasource.DataSourceHelper;
35 import com.google.visualization.datasource.DataSourceRequest;
36 import com.google.visualization.datasource.base.DataSourceException;
37 import com.google.visualization.datasource.base.ReasonType;
38 import com.google.visualization.datasource.base.ResponseStatus;
39 import com.google.visualization.datasource.base.StatusType;
40 import com.google.visualization.datasource.base.TypeMismatchException;
41 import com.google.visualization.datasource.datatable.ColumnDescription;
42 import com.google.visualization.datasource.datatable.DataTable;
43 import com.google.visualization.datasource.datatable.TableRow;
44 import com.google.visualization.datasource.datatable.value.DateTimeValue;
45 import com.google.visualization.datasource.datatable.value.NumberValue;
46 import com.google.visualization.datasource.datatable.value.ValueType;
47 import com.googlecode.objectify.Key;
48 import com.ibm.icu.util.GregorianCalendar;
49 import com.ibm.icu.util.TimeZone;
50 import java.io.IOException;
51 import java.math.BigDecimal;
52 import java.math.RoundingMode;
53 import java.util.ArrayList;
54 import java.util.Arrays;
55 import java.util.Calendar;
56 import java.util.Collection;
57 import java.util.HashMap;
58 import java.util.Iterator;
59 import java.util.List;
60 import java.util.Map;
61 import java.util.Objects;
62 import java.util.function.Predicate;
63 import java.util.logging.Level;
64 import java.util.stream.Collectors;
65 import java.util.stream.Stream;
66 import javax.servlet.RequestDispatcher;
67 import javax.servlet.ServletException;
68 import javax.servlet.http.HttpServletRequest;
69 import javax.servlet.http.HttpServletResponse;
70 
71 import org.joda.time.format.DateTimeFormat;
72 import org.joda.time.format.DateTimeFormatter;
73 
74 import static com.googlecode.objectify.ObjectifyService.ofy;
75 
76 /** Represents the servlet that is invoked on loading the coverage overview page. */
77 public class ShowCoverageOverviewServlet extends BaseServlet {
78 
79     @Override
getNavParentType()80     public PageType getNavParentType() {
81         return PageType.COVERAGE_OVERVIEW;
82     }
83 
84     @Override
getBreadcrumbLinks(HttpServletRequest request)85     public List<Page> getBreadcrumbLinks(HttpServletRequest request) {
86         return null;
87     }
88 
89     @Override
doGetHandler(HttpServletRequest request, HttpServletResponse response)90     public void doGetHandler(HttpServletRequest request, HttpServletResponse response)
91             throws IOException {
92 
93         String pageType =
94                 request.getParameter("pageType") == null
95                         ? "html"
96                         : request.getParameter("pageType");
97 
98         RequestDispatcher dispatcher;
99         if (pageType.equalsIgnoreCase("html")) {
100             dispatcher = this.getCoverageDispatcher(request, response);
101             try {
102                 request.setAttribute("pageType", pageType);
103                 response.setStatus(HttpServletResponse.SC_OK);
104                 dispatcher.forward(request, response);
105             } catch (ServletException e) {
106                 logger.log(Level.SEVERE, "Servlet Exception caught : ", e);
107             }
108         } else {
109 
110             String testName = request.getParameter("testName");
111 
112             DataTable data = getCoverageDataTable(testName);
113             DataSourceRequest dsRequest = null;
114 
115             try {
116                 // Extract the datasource request parameters.
117                 dsRequest = new DataSourceRequest(request);
118 
119                 // NOTE: If you want to work in restricted mode, which means that only
120                 // requests from the same domain can access the data source, uncomment the following
121                 // call.
122                 //
123                 // DataSourceHelper.verifyAccessApproved(dsRequest);
124 
125                 // Apply the query to the data table.
126                 DataTable newData =
127                         DataSourceHelper.applyQuery(
128                                 dsRequest.getQuery(), data, dsRequest.getUserLocale());
129 
130                 // Set the response.
131                 DataSourceHelper.setServletResponse(newData, dsRequest, response);
132             } catch (RuntimeException rte) {
133                 logger.log(Level.SEVERE, "A runtime exception has occured", rte);
134                 ResponseStatus status =
135                         new ResponseStatus(
136                                 StatusType.ERROR, ReasonType.INTERNAL_ERROR, rte.getMessage());
137                 if (dsRequest == null) {
138                     dsRequest = DataSourceRequest.getDefaultDataSourceRequest(request);
139                 }
140                 DataSourceHelper.setServletErrorResponse(status, dsRequest, response);
141             } catch (DataSourceException e) {
142                 if (dsRequest != null) {
143                     DataSourceHelper.setServletErrorResponse(e, dsRequest, response);
144                 } else {
145                     DataSourceHelper.setServletErrorResponse(e, request, response);
146                 }
147             }
148         }
149     }
150 
getTestRunEntityKeyList( List<TestCoverageStatusEntity> testCoverageStatusEntityList)151     private List<Key<TestRunEntity>> getTestRunEntityKeyList(
152             List<TestCoverageStatusEntity> testCoverageStatusEntityList) {
153         return testCoverageStatusEntityList.stream()
154                 .map(
155                         testCoverageStatusEntity -> {
156                             com.googlecode.objectify.Key testKey =
157                                     com.googlecode.objectify.Key.create(
158                                             TestEntity.class,
159                                             testCoverageStatusEntity.getTestName());
160                             return com.googlecode.objectify.Key.create(
161                                     testKey,
162                                     TestRunEntity.class,
163                                     testCoverageStatusEntity.getUpdatedTimestamp());
164                         })
165                 .collect(Collectors.toList());
166     }
167 
isBranchAndDevice(String branch, String device)168     private Predicate<DeviceInfoEntity> isBranchAndDevice(String branch, String device) {
169         return d -> d.getBranch().equals(branch) && d.getBuildFlavor().equals(device);
170     }
171 
isBranch(String branch)172     private Predicate<DeviceInfoEntity> isBranch(String branch) {
173         return d -> d.getBranch().equals(branch);
174     }
175 
isDevice(String device)176     private Predicate<DeviceInfoEntity> isDevice(String device) {
177         return d -> d.getBuildFlavor().equals(device);
178     }
179 
getCoverageDispatcher( HttpServletRequest request, HttpServletResponse response)180     private RequestDispatcher getCoverageDispatcher(
181             HttpServletRequest request, HttpServletResponse response) {
182 
183         String COVERAGE_OVERVIEW_JSP = "WEB-INF/jsp/show_coverage_overview.jsp";
184 
185         RequestDispatcher dispatcher = null;
186         boolean unfiltered = request.getParameter("unfiltered") != null;
187         boolean showPresubmit = request.getParameter("showPresubmit") != null;
188         boolean showPostsubmit = request.getParameter("showPostsubmit") != null;
189 
190         // If no params are specified, set to default of postsubmit-only.
191         if (!(showPresubmit || showPostsubmit)) {
192             showPostsubmit = true;
193         }
194 
195         // If unfiltered, set showPre- and Post-submit to true for accurate UI.
196         if (unfiltered) {
197             showPostsubmit = true;
198             showPresubmit = true;
199         }
200 
201         // Add test names to list
202         List<String> resultNames =
203                 Arrays.stream(VtsReportMessage.TestCaseResult.values())
204                         .map(testCaseResult -> testCaseResult.name())
205                         .collect(Collectors.toList());
206 
207         Map<String, String[]> parameterMap = request.getParameterMap();
208 
209         List<TestCoverageStatusEntity> testCoverageStatusEntityList =
210                 TestCoverageStatusEntity.getAllTestCoverage();
211 
212         List<com.googlecode.objectify.Key<TestRunEntity>> testRunEntityKeyList = new ArrayList<>();
213 
214         if (Objects.nonNull(parameterMap.get("branch"))
215                 || Objects.nonNull(parameterMap.get("device"))) {
216             List<com.googlecode.objectify.Key<DeviceInfoEntity>> deviceInfoEntityKeyList =
217                     TestCoverageStatusEntity.getDeviceInfoEntityKeyList(
218                             testCoverageStatusEntityList);
219 
220             Collection<DeviceInfoEntity> deviceInfoEntityMap =
221                     ofy().load().keys(() -> deviceInfoEntityKeyList.iterator()).values();
222 
223             Stream<DeviceInfoEntity> deviceInfoEntityStream = Stream.empty();
224             if (Objects.nonNull(parameterMap.get("branch"))
225                     && Objects.nonNull(parameterMap.get("device"))) {
226                 String branch = parameterMap.get("branch")[0];
227                 String device = parameterMap.get("device")[0];
228                 deviceInfoEntityStream =
229                         deviceInfoEntityMap.stream().filter(isBranchAndDevice(branch, device));
230             } else if (Objects.nonNull(parameterMap.get("branch"))) {
231                 String branch = parameterMap.get("branch")[0];
232                 deviceInfoEntityStream = deviceInfoEntityMap.stream().filter(isBranch(branch));
233             } else if (Objects.nonNull(parameterMap.get("device"))) {
234                 String device = parameterMap.get("device")[0];
235                 deviceInfoEntityStream = deviceInfoEntityMap.stream().filter(isDevice(device));
236             } else {
237                 logger.log(Level.WARNING, "unmet search condition!");
238             }
239             testRunEntityKeyList =
240                     deviceInfoEntityStream
241                             .map(
242                                     deviceInfoEntity -> {
243                                         com.googlecode.objectify.Key testKey =
244                                                 com.googlecode.objectify.Key.create(
245                                                         TestEntity.class,
246                                                         deviceInfoEntity
247                                                                 .getParent()
248                                                                 .getParent()
249                                                                 .getName());
250                                         return com.googlecode.objectify.Key.create(
251                                                 testKey,
252                                                 TestRunEntity.class,
253                                                 deviceInfoEntity.getParent().getId());
254                                     })
255                             .collect(Collectors.toList());
256             logger.log(Level.INFO, "testRunEntityKeyList size => " + testRunEntityKeyList.size());
257         } else {
258             testRunEntityKeyList = this.getTestRunEntityKeyList(testCoverageStatusEntityList);
259         }
260         Iterator<Key<TestRunEntity>> testRunEntityKeyIterator = testRunEntityKeyList.iterator();
261 
262         Map<Key<TestRunEntity>, TestRunEntity> keyTestRunEntityMap =
263                 ofy().load().keys(() -> testRunEntityKeyIterator);
264 
265         List<com.googlecode.objectify.Key<CodeCoverageEntity>> codeCoverageEntityKeyList =
266                 new ArrayList<>();
267         Map<Long, TestRunEntity> testRunEntityMap = new HashMap<>();
268         for (Map.Entry<com.googlecode.objectify.Key<TestRunEntity>, TestRunEntity> entry :
269                 keyTestRunEntityMap.entrySet()) {
270             com.googlecode.objectify.Key codeCoverageEntityKey =
271                     com.googlecode.objectify.Key.create(
272                             entry.getKey(), CodeCoverageEntity.class, entry.getValue().getId());
273             codeCoverageEntityKeyList.add(codeCoverageEntityKey);
274             testRunEntityMap.put(entry.getValue().getId(), entry.getValue());
275         }
276 
277         Map<com.googlecode.objectify.Key<CodeCoverageEntity>, CodeCoverageEntity>
278                 keyCodeCoverageEntityMap =
279                         ofy().load().keys(() -> codeCoverageEntityKeyList.iterator());
280 
281         Map<Long, CodeCoverageEntity> codeCoverageEntityMap = new HashMap<>();
282         for (Map.Entry<com.googlecode.objectify.Key<CodeCoverageEntity>, CodeCoverageEntity> entry :
283                 keyCodeCoverageEntityMap.entrySet()) {
284             codeCoverageEntityMap.put(entry.getValue().getId(), entry.getValue());
285         }
286 
287         int coveredLines = 0;
288         int uncoveredLines = 0;
289         int passCount = 0;
290         int failCount = 0;
291         for (Map.Entry<Long, CodeCoverageEntity> entry : codeCoverageEntityMap.entrySet()) {
292             TestRunEntity testRunEntity = testRunEntityMap.get(entry.getKey());
293 
294             CodeCoverageEntity codeCoverageEntity = entry.getValue();
295 
296             coveredLines += codeCoverageEntity.getCoveredLineCount();
297             uncoveredLines +=
298                     codeCoverageEntity.getTotalLineCount()
299                             - codeCoverageEntity.getCoveredLineCount();
300             passCount += testRunEntity.getPassCount();
301             failCount += testRunEntity.getFailCount();
302         }
303 
304         FilterUtil.setAttributes(request, parameterMap);
305 
306         int[] testStats = new int[VtsReportMessage.TestCaseResult.values().length];
307         testStats[VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_PASS.getNumber()] = passCount;
308         testStats[VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_FAIL.getNumber()] = failCount;
309 
310         response.setStatus(HttpServletResponse.SC_OK);
311         request.setAttribute("resultNames", resultNames);
312         request.setAttribute("resultNamesJson", new Gson().toJson(resultNames));
313         request.setAttribute("testRunEntityList", testRunEntityMap.values());
314         request.setAttribute("codeCoverageEntityMap", codeCoverageEntityMap);
315         request.setAttribute("coveredLines", new Gson().toJson(coveredLines));
316         request.setAttribute("uncoveredLines", new Gson().toJson(uncoveredLines));
317         request.setAttribute("testStats", new Gson().toJson(testStats));
318 
319         request.setAttribute("unfiltered", unfiltered);
320         request.setAttribute("showPresubmit", showPresubmit);
321         request.setAttribute("showPostsubmit", showPostsubmit);
322 
323         request.setAttribute(
324                 "deviceOptions",
325                 TestCoverageStatusEntity.getDeviceSet(testCoverageStatusEntityList));
326         request.setAttribute(
327                 "branchOptions",
328                 TestCoverageStatusEntity.getBranchSet(testCoverageStatusEntityList));
329         dispatcher = request.getRequestDispatcher(COVERAGE_OVERVIEW_JSP);
330         return dispatcher;
331     }
332 
getCoverageDataTable(String testName)333     private DataTable getCoverageDataTable(String testName) {
334 
335         Datastore datastore = DatastoreOptions.getDefaultInstance().getService();
336 
337         DataTable dataTable = new DataTable();
338         ArrayList<ColumnDescription> cd = new ArrayList<>();
339         ColumnDescription startDate =
340                 new ColumnDescription("startDate", ValueType.DATETIME, "Date");
341         startDate.setPattern("yyyy-MM-dd");
342         cd.add(startDate);
343         cd.add(
344                 new ColumnDescription(
345                         "coveredLineCount", ValueType.NUMBER, "Covered Source Code Line Count"));
346         cd.add(
347                 new ColumnDescription(
348                         "totalLineCount", ValueType.NUMBER, "Total Source Code Line Count"));
349         cd.add(new ColumnDescription("percentage", ValueType.NUMBER, "Coverage Ratio (%)"));
350 
351         dataTable.addColumns(cd);
352 
353         Calendar cal = Calendar.getInstance();
354         cal.add(Calendar.MONTH, -6);
355         Long startTime = cal.getTime().getTime() * 1000;
356         Long endTime = Calendar.getInstance().getTime().getTime() * 1000;
357 
358         com.google.cloud.datastore.Key startKey =
359                 datastore
360                         .newKeyFactory()
361                         .setKind(TestRunEntity.KIND)
362                         .addAncestors(
363                                 PathElement.of(TestEntity.KIND, testName),
364                                 PathElement.of(TestRunEntity.KIND, startTime))
365                         .newKey(startTime);
366 
367         com.google.cloud.datastore.Key endKey =
368                 datastore
369                         .newKeyFactory()
370                         .setKind(TestRunEntity.KIND)
371                         .addAncestors(
372                                 PathElement.of(TestEntity.KIND, testName),
373                                 PathElement.of(TestRunEntity.KIND, endTime))
374                         .newKey(endTime);
375 
376         Filter codeCoverageFilter =
377                 CompositeFilter.and(
378                         PropertyFilter.lt("__key__", endKey),
379                         PropertyFilter.gt("__key__", startKey));
380 
381         List<CodeCoverageEntity> codeCoverageEntityList =
382                 ofy().load()
383                         .type(CodeCoverageEntity.class)
384                         .filter(codeCoverageFilter)
385                         .limit(10)
386                         .list();
387 
388         DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
389         Map<String, List<CodeCoverageEntity>> codeCoverageEntityListMap =
390                 codeCoverageEntityList
391                         .stream()
392                         .collect(
393                                 Collectors.groupingBy(
394                                         v -> dateTimeFormatter.print(v.getId() / 1000)));
395 
396         codeCoverageEntityListMap.forEach(
397                 (key, entityList) -> {
398                     GregorianCalendar gCal = new GregorianCalendar();
399                     gCal.setTimeZone(TimeZone.getTimeZone("GMT"));
400                     gCal.setTimeInMillis(entityList.get(0).getId() / 1000);
401 
402                     Long sumCoveredLine =
403                             entityList.stream().mapToLong(val -> val.getCoveredLineCount()).sum();
404                     Long sumTotalLine =
405                             entityList.stream().mapToLong(val -> val.getTotalLineCount()).sum();
406                     float percentage = 0;
407                     if (sumTotalLine > 0) {
408                         BigDecimal coveredLineNum = new BigDecimal(sumCoveredLine);
409                         BigDecimal totalLineNum = new BigDecimal(sumTotalLine);
410                         BigDecimal totalPercent = new BigDecimal(100);
411                         percentage =
412                                 coveredLineNum
413                                         .multiply(totalPercent)
414                                         .divide(totalLineNum, 2, RoundingMode.HALF_DOWN)
415                                         .floatValue();
416                     }
417 
418                     TableRow tableRow = new TableRow();
419                     tableRow.addCell(new DateTimeValue(gCal));
420                     tableRow.addCell(new NumberValue(sumCoveredLine));
421                     tableRow.addCell(new NumberValue(sumTotalLine));
422                     tableRow.addCell(new NumberValue(percentage));
423                     try {
424                         dataTable.addRow(tableRow);
425                     } catch (TypeMismatchException e) {
426                         logger.log(Level.WARNING, "Invalid type! ");
427                     }
428                 });
429 
430         return dataTable;
431     }
432 }
433