1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you
5  * may not use this file except in compliance with the License. You may
6  * obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13  * implied. See the License for the specific language governing
14  * permissions and limitations under the License.
15  */
16 
17 package com.android.vts.job;
18 
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertNotNull;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23 
24 import com.android.vts.entity.DeviceInfoEntity;
25 import com.android.vts.entity.ProfilingPointEntity;
26 import com.android.vts.entity.ProfilingPointRunEntity;
27 import com.android.vts.entity.ProfilingPointSummaryEntity;
28 import com.android.vts.entity.TestEntity;
29 import com.android.vts.entity.TestRunEntity;
30 import com.android.vts.proto.VtsReportMessage;
31 import com.android.vts.util.ObjectifyTestBase;
32 import com.android.vts.util.StatSummary;
33 import com.android.vts.util.TimeUtil;
34 import com.google.appengine.api.datastore.DatastoreService;
35 import com.google.appengine.api.datastore.DatastoreServiceFactory;
36 import com.google.appengine.api.datastore.Entity;
37 import com.google.appengine.api.datastore.EntityNotFoundException;
38 import com.google.appengine.api.datastore.Key;
39 import com.google.appengine.api.datastore.KeyFactory;
40 import com.google.appengine.api.datastore.Query;
41 import com.google.appengine.api.taskqueue.dev.LocalTaskQueue;
42 import com.google.appengine.api.taskqueue.dev.QueueStateInfo;
43 import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig;
44 import com.google.appengine.tools.development.testing.LocalServiceTestHelper;
45 import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig;
46 import java.time.Instant;
47 import java.time.LocalDateTime;
48 import java.time.Month;
49 import java.time.ZonedDateTime;
50 import java.util.ArrayList;
51 import java.util.Arrays;
52 import java.util.Date;
53 import java.util.HashMap;
54 import java.util.HashSet;
55 import java.util.List;
56 import java.util.Set;
57 import java.util.concurrent.TimeUnit;
58 import org.apache.commons.math3.stat.descriptive.moment.Mean;
59 import org.junit.jupiter.api.Test;
60 import org.junit.jupiter.api.AfterEach;
61 import org.junit.jupiter.api.BeforeEach;
62 
63 public class VtsProfilingStatsJobServletTest extends ObjectifyTestBase {
64     private final LocalServiceTestHelper helper =
65             new LocalServiceTestHelper(
66                     new LocalDatastoreServiceTestConfig(),
67                     new LocalTaskQueueTestConfig()
68                             .setQueueXmlPath("src/main/webapp/WEB-INF/queue.xml"));
69     private static final double THRESHOLD = 1e-10;
70 
71     @BeforeEach
setUp()72     public void setUp() {
73         helper.setUp();
74     }
75 
76     @AfterEach
tearDown()77     public void tearDown() {
78         helper.tearDown();
79     }
80 
createProfilingRun()81     private static void createProfilingRun() {
82         Date d = new Date();
83         long time = TimeUnit.MILLISECONDS.toMicros(d.getTime());
84         long canonicalTime = VtsProfilingStatsJobServlet.getCanonicalTime(time);
85         String test = "test";
86         String profilingPointName = "profilingPoint";
87         String xLabel = "xLabel";
88         String yLabel = "yLabel";
89         VtsReportMessage.VtsProfilingType type =
90                 VtsReportMessage.VtsProfilingType.VTS_PROFILING_TYPE_UNLABELED_VECTOR;
91         VtsReportMessage.VtsProfilingRegressionMode mode =
92                 VtsReportMessage.VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING;
93 
94         Key testKey = KeyFactory.createKey(TestEntity.KIND, test);
95         Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, time);
96         Long[] valueArray = new Long[] {1l, 2l, 3l, 4l, 5l};
97         StatSummary stats =
98                 new StatSummary(
99                         "expected",
100                         VtsReportMessage.VtsProfilingRegressionMode.UNKNOWN_REGRESSION_MODE);
101         for (long value : valueArray) {
102             stats.updateStats(value);
103         }
104         Mean mean = new Mean();
105         List<Long> values = Arrays.asList(valueArray);
106         ProfilingPointRunEntity profilingPointRunEntity =
107                 new ProfilingPointRunEntity(
108                         testRunKey,
109                         profilingPointName,
110                         type.getNumber(),
111                         mode.getNumber(),
112                         null,
113                         values,
114                         xLabel,
115                         yLabel,
116                         null);
117 
118         String branch = "master";
119         String product = "product";
120         String flavor = "flavor";
121         String id = "12345";
122         String bitness = "64";
123         String abiName = "abi";
124         DeviceInfoEntity device =
125                 new DeviceInfoEntity(testRunKey, branch, product, flavor, id, bitness, abiName);
126     }
127 
128     /**
129      * Test that tasks are correctly scheduled on the queue.
130      *
131      * @throws InterruptedException
132      */
133     @Test
testTasksScheduled()134     public void testTasksScheduled() throws InterruptedException {
135         String[] testNames = new String[] {"test1", "test2", "test3"};
136         List<Key> testKeys = new ArrayList();
137         Set<Key> testKeySet = new HashSet<>();
138         String kind = "TEST";
139         for (String testName : testNames) {
140             Key key = KeyFactory.createKey(kind, testName);
141             testKeys.add(key);
142             testKeySet.add(key);
143         }
144         VtsProfilingStatsJobServlet.addTasks(testKeys);
145         Thread.sleep(1000); // wait one second (tasks are scheduled asychronously), must wait.
146         LocalTaskQueue taskQueue = LocalTaskQueueTestConfig.getLocalTaskQueue();
147         QueueStateInfo qsi = taskQueue.getQueueStateInfo().get(VtsProfilingStatsJobServlet.QUEUE);
148         assertNotNull(qsi);
149         assertEquals(testNames.length, qsi.getTaskInfo().size());
150 
151         int i = 0;
152         for (QueueStateInfo.TaskStateInfo taskStateInfo : qsi.getTaskInfo()) {
153             assertEquals(
154                     VtsProfilingStatsJobServlet.PROFILING_STATS_JOB_URL, taskStateInfo.getUrl());
155             assertEquals("POST", taskStateInfo.getMethod());
156             String body = taskStateInfo.getBody();
157             String[] parts = body.split("=");
158             assertEquals(2, parts.length);
159             assertEquals(VtsProfilingStatsJobServlet.PROFILING_POINT_KEY, parts[0]);
160             String keyString = parts[1];
161             Key profilingPointRunKey;
162             try {
163                 profilingPointRunKey = KeyFactory.stringToKey(keyString);
164             } catch (IllegalArgumentException e) {
165                 fail();
166                 return;
167             }
168             assertTrue(testKeys.contains(profilingPointRunKey));
169         }
170     }
171 
172     /** Test that canonical time is correctly derived from a timestamp in the middle of the day. */
173     @Test
testCanonicalTimeMidday()174     public void testCanonicalTimeMidday() {
175         int year = 2017;
176         Month month = Month.MAY;
177         int day = 28;
178         int hour = 14;
179         int minute = 30;
180         LocalDateTime now = LocalDateTime.of(year, month.getValue(), day, hour, minute);
181         ZonedDateTime zdt = ZonedDateTime.of(now, TimeUtil.PT_ZONE);
182         long time = TimeUnit.SECONDS.toMicros(zdt.toEpochSecond());
183         long canonicalTime = VtsProfilingStatsJobServlet.getCanonicalTime(time);
184         long canonicalTimeSec = TimeUnit.MICROSECONDS.toSeconds(canonicalTime);
185         ZonedDateTime canonical =
186                 ZonedDateTime.ofInstant(Instant.ofEpochSecond(canonicalTimeSec), TimeUtil.PT_ZONE);
187         assertEquals(month, canonical.getMonth());
188         assertEquals(day, canonical.getDayOfMonth());
189         assertEquals(0, canonical.getHour());
190         assertEquals(0, canonical.getMinute());
191     }
192 
193     /** Test that canonical time is correctly derived at the boundary of two days (midnight). */
194     @Test
testCanonicalTimeMidnight()195     public void testCanonicalTimeMidnight() {
196         int year = 2017;
197         Month month = Month.MAY;
198         int day = 28;
199         int hour = 0;
200         int minute = 0;
201         LocalDateTime now = LocalDateTime.of(year, month.getValue(), day, hour, minute);
202         ZonedDateTime zdt = ZonedDateTime.of(now, TimeUtil.PT_ZONE);
203         long time = TimeUnit.SECONDS.toMicros(zdt.toEpochSecond());
204         long canonicalTime = VtsProfilingStatsJobServlet.getCanonicalTime(time);
205         long canonicalTimeSec = TimeUnit.MICROSECONDS.toSeconds(canonicalTime);
206         ZonedDateTime canonical =
207                 ZonedDateTime.ofInstant(Instant.ofEpochSecond(canonicalTimeSec), TimeUtil.PT_ZONE);
208         assertEquals(zdt, canonical);
209     }
210 
211     /** Test that new summaries are created with a clean database. */
212     @Test
testNewSummary()213     public void testNewSummary() {
214         Date d = new Date();
215         long time = TimeUnit.MILLISECONDS.toMicros(d.getTime());
216         String test = "test";
217 
218         Key testKey = KeyFactory.createKey(TestEntity.KIND, test);
219         Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, time);
220         Long[] valueArray = new Long[] {1l, 2l, 3l, 4l, 5l};
221         StatSummary expected =
222                 new StatSummary(
223                         "expected",
224                         VtsReportMessage.VtsProfilingRegressionMode.UNKNOWN_REGRESSION_MODE);
225         for (long value : valueArray) {
226             expected.updateStats(value);
227         }
228         Mean mean = new Mean();
229         List<Long> values = Arrays.asList(valueArray);
230         ProfilingPointRunEntity profilingPointRunEntity =
231                 new ProfilingPointRunEntity(
232                         testRunKey,
233                         "profilingPoint",
234                         VtsReportMessage.VtsProfilingType.VTS_PROFILING_TYPE_UNLABELED_VECTOR_VALUE,
235                         VtsReportMessage.VtsProfilingRegressionMode
236                                 .VTS_REGRESSION_MODE_INCREASING_VALUE,
237                         null,
238                         values,
239                         "xLabel",
240                         "yLabel",
241                         null);
242 
243         DeviceInfoEntity device =
244                 new DeviceInfoEntity(
245                         testRunKey, "master", "product", "flavor", "12345", "64", "abi");
246 
247         List<DeviceInfoEntity> devices = new ArrayList<>();
248         devices.add(device);
249 
250         boolean result =
251                 VtsProfilingStatsJobServlet.updateSummaries(
252                         testKey, profilingPointRunEntity, devices, time);
253         assertTrue(result);
254 
255         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
256 
257         // Check profiling point entity
258         Key profilingPointKey = ProfilingPointEntity.createKey(test, profilingPointRunEntity.getName());
259         ProfilingPointEntity profilingPointEntity = null;
260         try {
261             Entity profilingPoint = datastore.get(profilingPointKey);
262             profilingPointEntity = ProfilingPointEntity.fromEntity(profilingPoint);
263         } catch (EntityNotFoundException exception) {
264             fail();
265         }
266         assertNotNull(profilingPointEntity);
267         assertEquals(profilingPointRunEntity.getName(), profilingPointEntity.getProfilingPointName());
268         assertEquals(profilingPointRunEntity.getXLabel(), profilingPointEntity.getXLabel());
269         assertEquals(profilingPointRunEntity.getYLabel(), profilingPointEntity.getYLabel());
270         assertEquals(profilingPointRunEntity.getType(), profilingPointEntity.getType());
271         assertEquals(profilingPointRunEntity.getRegressionMode(), profilingPointEntity.getRegressionMode());
272 
273         // Check all summary entities
274         Query q = new Query(ProfilingPointSummaryEntity.KIND).setAncestor(profilingPointKey);
275         for (Entity e : datastore.prepare(q).asIterable()) {
276             ProfilingPointSummaryEntity pps = ProfilingPointSummaryEntity.fromEntity(e);
277             assertNotNull(pps);
278             assertTrue(
279                     pps.getBranch().equals(device.getBranch())
280                             || pps.getBranch().equals(ProfilingPointSummaryEntity.ALL));
281             assertTrue(
282                     pps.getBuildFlavor().equals(ProfilingPointSummaryEntity.ALL)
283                             || pps.getBuildFlavor().equals(device.getBuildFlavor()));
284             assertEquals(expected.getCount(), pps.getGlobalStats().getCount());
285             assertEquals(expected.getMax(), pps.getGlobalStats().getMax(), THRESHOLD);
286             assertEquals(expected.getMin(), pps.getGlobalStats().getMin(), THRESHOLD);
287             assertEquals(expected.getMean(), pps.getGlobalStats().getMean(), THRESHOLD);
288             assertEquals(expected.getSumSq(), pps.getGlobalStats().getSumSq(), THRESHOLD);
289         }
290     }
291 
292     /** Test that existing summaries are updated correctly when a job pushes new profiling data. */
293     @Test
testUpdateSummary()294     public void testUpdateSummary() {
295         Date d = new Date();
296         long time = TimeUnit.MILLISECONDS.toMicros(d.getTime());
297         String test = "test2";
298 
299         Key testKey = KeyFactory.createKey(TestEntity.KIND, test);
300         Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, time);
301         Long[] valueArray = new Long[] {0l};
302         List<Long> values = Arrays.asList(valueArray);
303 
304         // Create a new profiling point run
305         ProfilingPointRunEntity profilingPointRunEntity =
306                 new ProfilingPointRunEntity(
307                         testRunKey,
308                         "profilingPoint2",
309                         VtsReportMessage.VtsProfilingType.VTS_PROFILING_TYPE_UNLABELED_VECTOR_VALUE,
310                         VtsReportMessage.VtsProfilingRegressionMode
311                                 .VTS_REGRESSION_MODE_INCREASING_VALUE,
312                         null,
313                         values,
314                         "xLabel",
315                         "yLabel",
316                         null);
317 
318         // Create a device for the run
319         String series = "";
320         DeviceInfoEntity device =
321                 new DeviceInfoEntity(
322                         testRunKey, "master", "product", "flavor", "12345", "64", "abi");
323 
324         List<DeviceInfoEntity> devices = new ArrayList<>();
325         devices.add(device);
326 
327         // Create the existing stats
328         Key profilingPointKey = ProfilingPointEntity.createKey(test, profilingPointRunEntity.getName());
329         StatSummary expected =
330                 new StatSummary(
331                         "label",
332                         0,
333                         10,
334                         5,
335                         100,
336                         10,
337                         VtsReportMessage.VtsProfilingRegressionMode.UNKNOWN_REGRESSION_MODE);
338         ProfilingPointSummaryEntity summary =
339                 new ProfilingPointSummaryEntity(
340                         profilingPointKey,
341                         expected,
342                         new ArrayList<>(),
343                         new HashMap<>(),
344                         device.getBranch(),
345                         device.getBuildFlavor(),
346                         series,
347                         time);
348 
349         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
350         datastore.put(summary.toEntity());
351 
352         // Update the summaries in the database
353         boolean result =
354                 VtsProfilingStatsJobServlet.updateSummaries(
355                         testKey, profilingPointRunEntity, devices, time);
356         assertTrue(result);
357 
358         // Calculate the expected stats with the values from the new run
359         for (long value : values) expected.updateStats(value);
360 
361         // Get the summary and check the values match what is expected
362         Key summaryKey =
363                 ProfilingPointSummaryEntity.createKey(
364                         profilingPointKey, device.getBranch(), device.getBuildFlavor(), series, time);
365         ProfilingPointSummaryEntity pps = null;
366         try {
367             Entity e = datastore.get(summaryKey);
368             pps = ProfilingPointSummaryEntity.fromEntity(e);
369         } catch (EntityNotFoundException e) {
370             fail();
371         }
372         assertNotNull(pps);
373         assertTrue(pps.getBranch().equals(device.getBranch()));
374         assertTrue(pps.getBuildFlavor().equals(device.getBuildFlavor()));
375         assertEquals(expected.getCount(), pps.getGlobalStats().getCount());
376         assertEquals(expected.getMax(), pps.getGlobalStats().getMax(), THRESHOLD);
377         assertEquals(expected.getMin(), pps.getGlobalStats().getMin(), THRESHOLD);
378         assertEquals(expected.getMean(), pps.getGlobalStats().getMean(), THRESHOLD);
379         assertEquals(expected.getSumSq(), pps.getGlobalStats().getSumSq(), THRESHOLD);
380     }
381 }
382