1 /**
2  * Copyright 2016 Google Inc. All Rights Reserved.
3  *
4  * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5  * except in compliance with the License. You may obtain a copy of the License at
6  *
7  * <p>http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * <p>Unless required by applicable law or agreed to in writing, software distributed under the
10  * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11  * express or implied. See the License for the specific language governing permissions and
12  * limitations under the License.
13  */
14 package com.android.vts.util;
15 
16 
17 import com.android.vts.entity.ApiCoverageEntity;
18 import com.android.vts.entity.BranchEntity;
19 import com.android.vts.entity.BuildTargetEntity;
20 import com.android.vts.entity.CodeCoverageEntity;
21 import com.android.vts.entity.CoverageEntity;
22 import com.android.vts.entity.DeviceInfoEntity;
23 import com.android.vts.entity.ProfilingPointRunEntity;
24 import com.android.vts.entity.TestCaseRunEntity;
25 import com.android.vts.entity.TestEntity;
26 import com.android.vts.entity.TestPlanEntity;
27 import com.android.vts.entity.TestPlanRunEntity;
28 import com.android.vts.entity.TestRunEntity;
29 import com.android.vts.entity.TestRunEntity.TestRunType;
30 import com.android.vts.job.VtsAlertJobServlet;
31 import com.android.vts.job.VtsCoverageAlertJobServlet;
32 import com.android.vts.job.VtsProfilingStatsJobServlet;
33 import com.android.vts.proto.VtsReportMessage.AndroidDeviceInfoMessage;
34 import com.android.vts.proto.VtsReportMessage.ApiCoverageReportMessage;
35 import com.android.vts.proto.VtsReportMessage.CoverageReportMessage;
36 import com.android.vts.proto.VtsReportMessage.HalInterfaceMessage;
37 import com.android.vts.proto.VtsReportMessage.LogMessage;
38 import com.android.vts.proto.VtsReportMessage.ProfilingReportMessage;
39 import com.android.vts.proto.VtsReportMessage.TestCaseReportMessage;
40 import com.android.vts.proto.VtsReportMessage.TestCaseResult;
41 import com.android.vts.proto.VtsReportMessage.TestPlanReportMessage;
42 import com.android.vts.proto.VtsReportMessage.TestReportMessage;
43 import com.android.vts.proto.VtsReportMessage.UrlResourceMessage;
44 import com.google.appengine.api.datastore.DatastoreFailureException;
45 import com.google.appengine.api.datastore.DatastoreService;
46 import com.google.appengine.api.datastore.DatastoreServiceFactory;
47 import com.google.appengine.api.datastore.DatastoreTimeoutException;
48 import com.google.appengine.api.datastore.Entity;
49 import com.google.appengine.api.datastore.EntityNotFoundException;
50 import com.google.appengine.api.datastore.FetchOptions;
51 import com.google.appengine.api.datastore.Key;
52 import com.google.appengine.api.datastore.KeyFactory;
53 import com.google.appengine.api.datastore.Query;
54 import com.google.appengine.api.datastore.Query.Filter;
55 import com.google.appengine.api.datastore.Query.FilterOperator;
56 import com.google.appengine.api.datastore.Query.FilterPredicate;
57 import com.google.appengine.api.datastore.Transaction;
58 import com.google.appengine.api.datastore.TransactionOptions;
59 import com.google.common.collect.Lists;
60 import java.util.ArrayList;
61 import java.util.Arrays;
62 import java.util.ConcurrentModificationException;
63 import java.util.HashSet;
64 import java.util.List;
65 import java.util.Map;
66 import java.util.Set;
67 import java.util.logging.Level;
68 import java.util.logging.Logger;
69 import java.util.stream.Collectors;
70 
71 /**
72  * DatastoreHelper, a helper class for interacting with Cloud Datastore.
73  */
74 public class DatastoreHelper {
75 
76   /**
77    * The default kind name for datastore
78    */
79   public static final String NULL_ENTITY_KIND = "nullEntity";
80 
81   public static final int MAX_WRITE_RETRIES = 5;
82   /**
83    * This variable is for maximum number of entities per transaction You can find the detail here
84    * (https://cloud.google.com/datastore/docs/concepts/limits)
85    */
86   public static final int MAX_ENTITY_SIZE_PER_TRANSACTION = 300;
87 
88   protected static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName());
89   private static final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
90 
91   /**
92    * Get query fetch options for large batches of entities.
93    *
94    * @return FetchOptions with a large chunk and prefetch size.
95    */
getLargeBatchOptions()96   public static FetchOptions getLargeBatchOptions() {
97     return FetchOptions.Builder.withChunkSize(1000).prefetchSize(1000);
98   }
99 
100   /**
101    * Returns true if there are data points newer than lowerBound in the results table.
102    *
103    * @param parentKey The parent key to use in the query.
104    * @param kind The query entity kind.
105    * @param lowerBound The (exclusive) lower time bound, long, microseconds.
106    * @return boolean True if there are newer data points.
107    */
hasNewer(Key parentKey, String kind, Long lowerBound)108   public static boolean hasNewer(Key parentKey, String kind, Long lowerBound) {
109     if (lowerBound == null || lowerBound <= 0) {
110       return false;
111     }
112     DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
113     Key startKey = KeyFactory.createKey(parentKey, kind, lowerBound);
114     Filter startFilter =
115             new FilterPredicate(
116                     Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey);
117     Query q = new Query(kind).setAncestor(parentKey).setFilter(startFilter).setKeysOnly();
118     return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0;
119   }
120 
121   /**
122    * Returns true if there are data points older than upperBound in the table.
123    *
124    * @param parentKey The parent key to use in the query.
125    * @param kind The query entity kind.
126    * @param upperBound The (exclusive) upper time bound, long, microseconds.
127    * @return boolean True if there are older data points.
128    */
hasOlder(Key parentKey, String kind, Long upperBound)129   public static boolean hasOlder(Key parentKey, String kind, Long upperBound) {
130     if (upperBound == null || upperBound <= 0) {
131       return false;
132     }
133     Key endKey = KeyFactory.createKey(parentKey, kind, upperBound);
134     Filter endFilter =
135             new FilterPredicate(Entity.KEY_RESERVED_PROPERTY, FilterOperator.LESS_THAN, endKey);
136     Query q = new Query(kind).setAncestor(parentKey).setFilter(endFilter).setKeysOnly();
137     return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0;
138   }
139 
140   /**
141    * Get all of the devices branches.
142    *
143    * @return a list of all branches.
144    */
getAllBranches()145   public static List<String> getAllBranches() {
146     Query query = new Query(BranchEntity.KIND).setKeysOnly();
147     List<String> branches = new ArrayList<>();
148     for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) {
149       branches.add(e.getKey().getName());
150     }
151     return branches;
152   }
153 
154   /**
155    * Get all of the device build flavors.
156    *
157    * @return a list of all device build flavors.
158    */
getAllBuildFlavors()159   public static List<String> getAllBuildFlavors() {
160     Query query = new Query(BuildTargetEntity.KIND).setKeysOnly();
161     List<String> devices = new ArrayList<>();
162     for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) {
163       devices.add(e.getKey().getName());
164     }
165     return devices;
166   }
167 
168   /**
169    * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times and withXG of
170    * false value
171    *
172    * @param entity The entity that you want to insert to datastore.
173    * @param entityList The list of entity for using datastore put method.
174    */
datastoreTransactionalRetry(Entity entity, List<Entity> entityList)175   private static boolean datastoreTransactionalRetry(Entity entity, List<Entity> entityList) {
176     return datastoreTransactionalRetryWithXG(entity, entityList, false);
177   }
178 
179   /**
180    * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times
181    *
182    * @param entity The entity that you want to insert to datastore.
183    * @param entityList The list of entity for using datastore put method.
184    */
datastoreTransactionalRetryWithXG( Entity entity, List<Entity> entityList, boolean withXG)185   private static boolean datastoreTransactionalRetryWithXG(
186           Entity entity, List<Entity> entityList, boolean withXG) {
187     int retries = 0;
188     while (true) {
189       Transaction txn;
190       if (withXG) {
191         TransactionOptions options = TransactionOptions.Builder.withXG(withXG);
192         txn = datastore.beginTransaction(options);
193       } else {
194         txn = datastore.beginTransaction();
195       }
196 
197       try {
198         // Check if test already exists in the database
199         if (!entity.getKind().equalsIgnoreCase(NULL_ENTITY_KIND)) {
200           try {
201             if (entity.getKind().equalsIgnoreCase("Test")) {
202               Entity datastoreEntity = datastore.get(entity.getKey());
203               TestEntity datastoreTestEntity = TestEntity.fromEntity(datastoreEntity);
204               if (datastoreTestEntity == null
205                       || !datastoreTestEntity.equals(entity)) {
206                 entityList.add(entity);
207               }
208             } else if (entity.getKind().equalsIgnoreCase("TestPlan")) {
209               datastore.get(entity.getKey());
210             } else {
211               datastore.get(entity.getKey());
212             }
213           } catch (EntityNotFoundException e) {
214             entityList.add(entity);
215           }
216         }
217         datastore.put(txn, entityList);
218         txn.commit();
219         break;
220       } catch (ConcurrentModificationException
221               | DatastoreFailureException
222               | DatastoreTimeoutException e) {
223         entityList.remove(entity);
224         logger.log(
225                 Level.WARNING,
226                 "Retrying insert kind: " + entity.getKind() + " key: " + entity.getKey());
227         if (retries++ >= MAX_WRITE_RETRIES) {
228           logger.log(
229                   Level.SEVERE,
230                   "Exceeded maximum retries kind: "
231                           + entity.getKind()
232                           + " key: "
233                           + entity.getKey());
234           return false;
235         }
236       } finally {
237         if (txn.isActive()) {
238           logger.log(
239                   Level.WARNING, "Transaction rollback forced for : " + entity.getKind());
240           txn.rollback();
241         }
242       }
243     }
244     return true;
245   }
246 }
247