1 /*
2  * Copyright (c) 2018 Google Inc. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you
5  * may not use this file except in compliance with the License. You may
6  * obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13  * implied. See the License for the specific language governing
14  * permissions and limitations under the License.
15  */
16 
17 package com.android.vts.job;
18 
19 import com.android.vts.entity.TestSuiteFileEntity;
20 import com.android.vts.entity.TestSuiteResultEntity;
21 import com.android.vts.proto.TestSuiteResultMessageProto;
22 import com.android.vts.util.GcsHelper;
23 import com.android.vts.util.TaskQueueHelper;
24 import com.android.vts.util.TimeUtil;
25 import com.google.appengine.api.memcache.ErrorHandlers;
26 import com.google.appengine.api.memcache.MemcacheService;
27 import com.google.appengine.api.memcache.MemcacheServiceFactory;
28 import com.google.appengine.api.taskqueue.Queue;
29 import com.google.appengine.api.taskqueue.QueueFactory;
30 import com.google.appengine.api.taskqueue.TaskOptions;
31 import com.google.cloud.storage.Blob;
32 import com.google.cloud.storage.Bucket;
33 import com.google.cloud.storage.Storage;
34 import com.googlecode.objectify.Key;
35 import javax.servlet.ServletConfig;
36 import javax.servlet.ServletException;
37 import javax.servlet.http.HttpServletRequest;
38 import javax.servlet.http.HttpServletResponse;
39 import java.io.IOException;
40 import java.io.InputStream;
41 import java.nio.file.FileSystems;
42 import java.nio.file.Path;
43 import java.nio.file.Paths;
44 import java.time.ZonedDateTime;
45 import java.time.format.DateTimeFormatter;
46 import java.util.ArrayList;
47 import java.util.Arrays;
48 import java.util.Iterator;
49 import java.util.List;
50 import java.util.Objects;
51 import java.util.Optional;
52 import java.util.concurrent.TimeUnit;
53 import java.util.logging.Level;
54 import java.util.logging.Logger;
55 import java.util.stream.Collectors;
56 
57 import static com.googlecode.objectify.ObjectifyService.ofy;
58 
59 /** Suite Test result file processing job. */
60 public class VtsSuiteTestJobServlet extends BaseJobServlet {
61 
62     private static final String SUITE_TEST_URL = "/cron/test_suite_report_gcs_monitor";
63 
64     private static final String SERVICE_NAME = "VTS Dashboard";
65 
66     private final Logger logger = Logger.getLogger(this.getClass().getName());
67 
68     /** Google Cloud Storage project's key file to access the storage */
69     private static String GCS_KEY_FILE;
70     /** Google Cloud Storage project's default bucket name for vtslab log files */
71     private static String GCS_BUCKET_NAME;
72     /** Google Cloud Storage project's default directory name for suite test result files */
73     private static String GCS_SUITE_TEST_FOLDER_NAME;
74 
75     public static final String QUEUE = "suiteTestQueue";
76 
77     /**
78      * This is the key file to access vtslab-gcs project. It will allow the dashboard to have a full
79      * control of the bucket.
80      */
81     private InputStream keyFileInputStream;
82 
83     /** This is the instance of java google storage library */
84     private Storage storage;
85 
86     /** This is the instance of App Engine memcache service java library */
87     private MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService();
88 
89     @Override
init(ServletConfig servletConfig)90     public void init(ServletConfig servletConfig) throws ServletException {
91         super.init(servletConfig);
92 
93         GCS_KEY_FILE = systemConfigProp.getProperty("gcs.keyFile");
94         GCS_BUCKET_NAME = systemConfigProp.getProperty("gcs.bucketName");
95         GCS_SUITE_TEST_FOLDER_NAME = systemConfigProp.getProperty("gcs.suiteTestFolderName");
96 
97         this.keyFileInputStream =
98                 this.getClass().getClassLoader().getResourceAsStream("keys/" + GCS_KEY_FILE);
99 
100         Optional<Storage> optionalStorage = GcsHelper.getStorage(this.keyFileInputStream);
101         if (optionalStorage.isPresent()) {
102             this.storage = optionalStorage.get();
103         } else {
104             logger.log(Level.SEVERE, "Error on getting storage instance!");
105             throw new ServletException("Creating storage instance exception!");
106         }
107         syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
108     }
109 
110     @Override
doGet(HttpServletRequest request, HttpServletResponse response)111     public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
112 
113         List<String> dateStringList = new ArrayList<>();
114 
115         long currentMicroSecond = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis());
116 
117         ZonedDateTime checkZonedDateTime = TimeUtil.getZonedDateTime(currentMicroSecond);
118         String checkDateString =
119                 DateTimeFormatter.ofPattern(TimeUtil.DATE_FORMAT)
120                         .format(checkZonedDateTime.minusMinutes(5));
121         String todayDateString = TimeUtil.getDateString(currentMicroSecond);
122         if (!checkDateString.equals(todayDateString)) {
123             dateStringList.add(checkDateString);
124             logger.log(Level.INFO, "Yesterday is added to the process queue and processed!");
125         }
126         dateStringList.add(todayDateString);
127 
128         for (String dateString : dateStringList) {
129             String[] dateArray = dateString.split("-");
130             if (dateArray.length == 3) {
131 
132                 Queue queue = QueueFactory.getQueue(QUEUE);
133 
134                 List<TaskOptions> tasks = new ArrayList<>();
135 
136                 String fileSeparator = FileSystems.getDefault().getSeparator();
137 
138                 String year = dateArray[0];
139                 String month = dateArray[1];
140                 String day = dateArray[2];
141 
142                 List<String> pathList = Arrays.asList(GCS_SUITE_TEST_FOLDER_NAME, year, month, day);
143                 Path pathInfo = Paths.get(String.join(fileSeparator, pathList));
144 
145                 List<TestSuiteFileEntity> testSuiteFileEntityList =
146                         ofy().load()
147                                 .type(TestSuiteFileEntity.class)
148                                 .filter("year", Integer.parseInt(year))
149                                 .filter("month", Integer.parseInt(month))
150                                 .filter("day", Integer.parseInt(day))
151                                 .list();
152 
153                 List<String> filePathList =
154                         testSuiteFileEntityList
155                                 .stream()
156                                 .map(testSuiteFile -> testSuiteFile.getFilePath())
157                                 .collect(Collectors.toList());
158 
159                 Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME);
160 
161                 Storage.BlobListOption[] listOptions =
162                         new Storage.BlobListOption[] {
163                             Storage.BlobListOption.prefix(pathInfo.toString() + fileSeparator)
164                         };
165 
166                 Iterable<Blob> blobIterable = vtsReportBucket.list(listOptions).iterateAll();
167                 Iterator<Blob> blobIterator = blobIterable.iterator();
168                 while (blobIterator.hasNext()) {
169                     Blob blob = blobIterator.next();
170                     if (blob.isDirectory()) {
171                         logger.log(Level.INFO, blob.getName() + " directory will be skipped!");
172                     } else {
173                         if (filePathList.contains(blob.getName())) {
174                             logger.log(Level.INFO, "filePathList contain => " + blob.getName());
175                         } else if (blob.getName().endsWith(fileSeparator)) {
176                             logger.log(Level.INFO, blob.getName() + " endswith slash!");
177                         } else {
178                             TaskOptions task =
179                                     TaskOptions.Builder.withUrl(SUITE_TEST_URL)
180                                             .param("filePath", blob.getName())
181                                             .method(TaskOptions.Method.POST);
182                             tasks.add(task);
183                         }
184                     }
185                 }
186                 TaskQueueHelper.addToQueue(queue, tasks);
187             } else {
188                 throw new IllegalArgumentException(
189                         todayDateString + " date string not in correct format");
190             }
191         }
192     }
193 
194     @Override
doPost(HttpServletRequest request, HttpServletResponse response)195     public void doPost(HttpServletRequest request, HttpServletResponse response)
196             throws IOException {
197 
198         String filePath = request.getParameter("filePath");
199         if (Objects.isNull(filePath)) {
200             logger.log(Level.WARNING, "filePath parameter is null!");
201         } else {
202             logger.log(Level.INFO, "filePath parameter => " + filePath);
203 
204             Key<TestSuiteFileEntity> testSuiteFileEntityKey =
205                     Key.create(TestSuiteFileEntity.class, filePath);
206             TestSuiteFileEntity testSuiteFileEntity =
207                     ofy().load()
208                             .type(TestSuiteFileEntity.class)
209                             .filterKey(testSuiteFileEntityKey)
210                             .first()
211                             .now();
212 
213             if (Objects.isNull(testSuiteFileEntity)) {
214                 Blob blobFile = (Blob) this.syncCache.get(filePath);
215                 if (Objects.isNull(blobFile)) {
216                     Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME);
217                     blobFile = vtsReportBucket.get(filePath);
218                     this.syncCache.put(filePath, blobFile);
219                 }
220 
221                 if (blobFile.exists()) {
222                     TestSuiteFileEntity newTestSuiteFileEntity = new TestSuiteFileEntity(filePath);
223                     try {
224                         TestSuiteResultMessageProto.TestSuiteResultMessage testSuiteResultMessage =
225                                 TestSuiteResultMessageProto.TestSuiteResultMessage.parseFrom(
226                                         blobFile.getContent());
227 
228                         TestSuiteResultEntity testSuiteResultEntity =
229                                 new TestSuiteResultEntity(
230                                         testSuiteFileEntityKey,
231                                         testSuiteResultMessage.getStartTime(),
232                                         testSuiteResultMessage.getEndTime(),
233                                         testSuiteResultMessage.getTestType(),
234                                         testSuiteResultMessage.getBootSuccess(),
235                                         testSuiteResultMessage.getResultPath(),
236                                         testSuiteResultMessage.getInfraLogPath(),
237                                         testSuiteResultMessage.getHostName(),
238                                         testSuiteResultMessage.getSuitePlan(),
239                                         testSuiteResultMessage.getSuiteVersion(),
240                                         testSuiteResultMessage.getSuiteName(),
241                                         testSuiteResultMessage.getSuiteBuildNumber(),
242                                         testSuiteResultMessage.getModulesDone(),
243                                         testSuiteResultMessage.getModulesTotal(),
244                                         testSuiteResultMessage.getBranch(),
245                                         testSuiteResultMessage.getTarget(),
246                                         testSuiteResultMessage.getBuildId(),
247                                         testSuiteResultMessage.getBuildSystemFingerprint(),
248                                         testSuiteResultMessage.getBuildVendorFingerprint(),
249                                         testSuiteResultMessage.getPassedTestCaseCount(),
250                                         testSuiteResultMessage.getFailedTestCaseCount());
251 
252                         testSuiteResultEntity.save(newTestSuiteFileEntity);
253                     } catch (IOException e) {
254                         ofy().delete().type(TestSuiteFileEntity.class).id(filePath).now();
255                         response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
256                         logger.log(Level.WARNING, "Invalid proto: " + e.getLocalizedMessage());
257                     }
258                 }
259             } else {
260                 logger.log(Level.INFO, "suite test found in datastore!");
261             }
262         }
263     }
264 }
265