1 /*
2  * Copyright (c) 2016 Google Inc. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you
5  * may not use this file except in compliance with the License. You may
6  * obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13  * implied. See the License for the specific language governing
14  * permissions and limitations under the License.
15  */
16 
17 package com.android.vts.job;
18 
19 import com.android.vts.entity.DeviceInfoEntity;
20 import com.android.vts.entity.TestAcknowledgmentEntity;
21 import com.android.vts.entity.TestCaseRunEntity;
22 import com.android.vts.entity.TestCaseRunEntity.TestCase;
23 import com.android.vts.entity.TestRunEntity;
24 import com.android.vts.entity.TestStatusEntity;
25 import com.android.vts.entity.TestStatusEntity.TestCaseReference;
26 import com.android.vts.proto.VtsReportMessage.TestCaseResult;
27 import com.android.vts.util.DatastoreHelper;
28 import com.android.vts.util.EmailHelper;
29 import com.android.vts.util.FilterUtil;
30 import com.android.vts.util.TimeUtil;
31 import com.google.appengine.api.datastore.DatastoreFailureException;
32 import com.google.appengine.api.datastore.DatastoreService;
33 import com.google.appengine.api.datastore.DatastoreServiceFactory;
34 import com.google.appengine.api.datastore.DatastoreTimeoutException;
35 import com.google.appengine.api.datastore.Entity;
36 import com.google.appengine.api.datastore.EntityNotFoundException;
37 import com.google.appengine.api.datastore.FetchOptions;
38 import com.google.appengine.api.datastore.Key;
39 import com.google.appengine.api.datastore.KeyFactory;
40 import com.google.appengine.api.datastore.Query;
41 import com.google.appengine.api.datastore.Query.Filter;
42 import com.google.appengine.api.datastore.Query.SortDirection;
43 import com.google.appengine.api.datastore.Transaction;
44 import com.google.appengine.api.taskqueue.Queue;
45 import com.google.appengine.api.taskqueue.QueueFactory;
46 import com.google.appengine.api.taskqueue.TaskOptions;
47 import java.io.IOException;
48 import java.io.UnsupportedEncodingException;
49 import java.util.ArrayList;
50 import java.util.Comparator;
51 import java.util.ConcurrentModificationException;
52 import java.util.HashMap;
53 import java.util.HashSet;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.Set;
57 import java.util.concurrent.TimeUnit;
58 import java.util.logging.Level;
59 import java.util.logging.Logger;
60 import javax.mail.Message;
61 import javax.mail.MessagingException;
62 import javax.servlet.http.HttpServletRequest;
63 import javax.servlet.http.HttpServletResponse;
64 import org.apache.commons.lang.StringUtils;
65 
66 /** Represents the notifications service which is automatically called on a fixed schedule. */
67 public class VtsAlertJobServlet extends BaseJobServlet {
68     private static final String ALERT_JOB_URL = "/task/vts_alert_job";
69     protected static final Logger logger = Logger.getLogger(VtsAlertJobServlet.class.getName());
70     protected static final int MAX_RUN_COUNT = 1000; // maximum number of runs to query for
71 
72     /**
73      * Process the current test case failures for a test.
74      *
75      * @param status The TestStatusEntity object for the test.
76      * @returns a map from test case name to the test case run ID for which the test case failed.
77      */
getCurrentFailures(TestStatusEntity status)78     private static Map<String, TestCase> getCurrentFailures(TestStatusEntity status) {
79         if (status.getFailingTestCases() == null || status.getFailingTestCases().size() == 0) {
80             return new HashMap<>();
81         }
82         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
83         Map<String, TestCase> failingTestcases = new HashMap<>();
84         Set<Key> gets = new HashSet<>();
85         for (TestCaseReference testCaseRef : status.getFailingTestCases()) {
86             gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId));
87         }
88         if (gets.size() == 0) {
89             return failingTestcases;
90         }
91         Map<Key, Entity> testCaseMap = datastore.get(gets);
92 
93         for (TestCaseReference testCaseRef : status.getFailingTestCases()) {
94             Key key = KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId);
95             if (!testCaseMap.containsKey(key)) {
96                 continue;
97             }
98             Entity testCaseRun = testCaseMap.get(key);
99             TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun);
100             if (testCaseRunEntity.testCases.size() <= testCaseRef.offset) {
101                 continue;
102             }
103             TestCase testCase = testCaseRunEntity.testCases.get(testCaseRef.offset);
104             failingTestcases.put(testCase.name, testCase);
105         }
106         return failingTestcases;
107     }
108 
109     /**
110      * Get the test acknowledgments for a test key.
111      *
112      * @param testKey The key to the test whose acknowledgments to fetch.
113      * @return A list of test acknowledgments.
114      */
getTestCaseAcknowledgments(Key testKey)115     private static List<TestAcknowledgmentEntity> getTestCaseAcknowledgments(Key testKey) {
116         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
117 
118         List<TestAcknowledgmentEntity> acks = new ArrayList<>();
119         Filter testFilter =
120                 new Query.FilterPredicate(
121                         TestAcknowledgmentEntity.TEST_KEY, Query.FilterOperator.EQUAL, testKey);
122         Query q = new Query(TestAcknowledgmentEntity.KIND).setFilter(testFilter);
123 
124         for (Entity ackEntity : datastore.prepare(q).asIterable()) {
125             TestAcknowledgmentEntity ack = TestAcknowledgmentEntity.fromEntity(ackEntity);
126             if (ack == null) continue;
127             acks.add(ack);
128         }
129         return acks;
130     }
131 
132     /**
133      * Get the test runs for the test in the specified time window.
134      *
135      * <p>If the start and end time delta is greater than one day, the query will be truncated.
136      *
137      * @param testKey The key to the test whose runs to query.
138      * @param startTime The start time for the query.
139      * @param endTime The end time for the query.
140      * @return A list of test runs in the specified time window.
141      */
getTestRuns(Key testKey, long startTime, long endTime)142     private static List<TestRunEntity> getTestRuns(Key testKey, long startTime, long endTime) {
143         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
144         Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false);
145         long delta = endTime - startTime;
146         delta = Math.min(delta, TimeUnit.DAYS.toMicros(1));
147         Filter runFilter =
148                 FilterUtil.getTimeFilter(
149                         testKey, TestRunEntity.KIND, endTime - delta + 1, endTime, testTypeFilter);
150 
151         Query q =
152                 new Query(TestRunEntity.KIND)
153                         .setAncestor(testKey)
154                         .setFilter(runFilter)
155                         .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING);
156 
157         List<TestRunEntity> testRuns = new ArrayList<>();
158         for (Entity testRunEntity :
159                 datastore.prepare(q).asIterable(FetchOptions.Builder.withLimit(MAX_RUN_COUNT))) {
160             TestRunEntity testRun = TestRunEntity.fromEntity(testRunEntity);
161             if (testRun == null) continue;
162             testRuns.add(testRun);
163         }
164         return testRuns;
165     }
166 
167     /**
168      * Separate the test cases which are acknowledged by the provided acknowledgments.
169      *
170      * @param testCases The list of test case names.
171      * @param devices The list of devices for a test run.
172      * @param acks The list of acknowledgments for the test.
173      * @return A list of acknowledged test case names that have been removed from the input test
174      *     cases.
175      */
separateAcknowledged( Set<String> testCases, List<DeviceInfoEntity> devices, List<TestAcknowledgmentEntity> acks)176     public static Set<String> separateAcknowledged(
177             Set<String> testCases,
178             List<DeviceInfoEntity> devices,
179             List<TestAcknowledgmentEntity> acks) {
180         Set<String> acknowledged = new HashSet<>();
181         for (TestAcknowledgmentEntity ack : acks) {
182             boolean allDevices = ack.getDevices() == null || ack.getDevices().size() == 0;
183             boolean allBranches = ack.getBranches() == null || ack.getBranches().size() == 0;
184             boolean isRelevant = allDevices && allBranches;
185 
186             // Determine if the acknowledgment is relevant to the devices.
187             if (!isRelevant) {
188                 for (DeviceInfoEntity device : devices) {
189                     boolean deviceAcknowledged =
190                             allDevices || ack.getDevices().contains(device.getBuildFlavor());
191                     boolean branchAcknowledged =
192                             allBranches || ack.getBranches().contains(device.getBranch());
193                     if (deviceAcknowledged && branchAcknowledged) isRelevant = true;
194                 }
195             }
196 
197             if (isRelevant) {
198                 // Separate the test cases
199                 boolean allTestCases =
200                         ack.getTestCaseNames() == null || ack.getTestCaseNames().size() == 0;
201                 if (allTestCases) {
202                     acknowledged.addAll(testCases);
203                     testCases.removeAll(acknowledged);
204                 } else {
205                     for (String testCase : ack.getTestCaseNames()) {
206                         if (testCases.contains(testCase)) {
207                             acknowledged.add(testCase);
208                             testCases.remove(testCase);
209                         }
210                     }
211                 }
212             }
213         }
214         return acknowledged;
215     }
216 
217     /**
218      * Checks whether any new failures have occurred beginning since (and including) startTime.
219      *
220      * @param testRuns The list of test runs for which to update the status.
221      * @param link The string URL linking to the test's status table.
222      * @param failedTestCaseMap The map of test case names to TestCase for those failing in the last
223      *     status update.
224      * @param emailAddresses The list of email addresses to send notifications to.
225      * @param messages The email Message queue.
226      * @returns latest TestStatusMessage or null if no update is available.
227      * @throws IOException
228      */
getTestStatus( List<TestRunEntity> testRuns, String link, Map<String, TestCase> failedTestCaseMap, List<TestAcknowledgmentEntity> testAcks, List<String> emailAddresses, List<Message> messages)229     public TestStatusEntity getTestStatus(
230             List<TestRunEntity> testRuns,
231             String link,
232             Map<String, TestCase> failedTestCaseMap,
233             List<TestAcknowledgmentEntity> testAcks,
234             List<String> emailAddresses,
235             List<Message> messages)
236             throws IOException {
237         if (testRuns.size() == 0) return null;
238         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
239 
240         TestRunEntity mostRecentRun = null;
241         Map<String, TestCaseResult> mostRecentTestCaseResults = new HashMap<>();
242         Map<String, TestCase> testCaseBreakageMap = new HashMap<>();
243         int passingTestcaseCount = 0;
244         List<TestCaseReference> failingTestCases = new ArrayList<>();
245         Set<String> fixedTestcases = new HashSet<>();
246         Set<String> newTestcaseFailures = new HashSet<>();
247         Set<String> continuedTestcaseFailures = new HashSet<>();
248         Set<String> skippedTestcaseFailures = new HashSet<>();
249         Set<String> transientTestcaseFailures = new HashSet<>();
250 
251         for (TestRunEntity testRun : testRuns) {
252             if (mostRecentRun == null) {
253                 mostRecentRun = testRun;
254             }
255             List<Key> testCaseKeys = new ArrayList<>();
256             for (long testCaseId : testRun.getTestCaseIds()) {
257                 testCaseKeys.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId));
258             }
259             Map<Key, Entity> entityMap = datastore.get(testCaseKeys);
260             for (Key testCaseKey : testCaseKeys) {
261                 if (!entityMap.containsKey(testCaseKey)) {
262                     logger.log(Level.WARNING, "Test case entity missing: " + testCaseKey);
263                     continue;
264                 }
265                 Entity testCaseRun = entityMap.get(testCaseKey);
266                 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun);
267                 if (testCaseRunEntity == null) {
268                     logger.log(Level.WARNING, "Invalid test case run: " + testCaseRun.getKey());
269                     continue;
270                 }
271                 for (TestCase testCase : testCaseRunEntity.testCases) {
272                     String testCaseName = testCase.name;
273                     TestCaseResult result = TestCaseResult.valueOf(testCase.result);
274 
275                     if (mostRecentRun == testRun) {
276                         mostRecentTestCaseResults.put(testCaseName, result);
277                     } else {
278                         if (!mostRecentTestCaseResults.containsKey(testCaseName)) {
279                             // Deprecate notifications for tests that are not present on newer runs
280                             continue;
281                         }
282                         TestCaseResult mostRecentRes = mostRecentTestCaseResults.get(testCaseName);
283                         if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_SKIP) {
284                             mostRecentTestCaseResults.put(testCaseName, result);
285                         } else if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_PASS) {
286                             // Test is passing now, witnessed a transient failure
287                             if (result != TestCaseResult.TEST_CASE_RESULT_PASS
288                                     && result != TestCaseResult.TEST_CASE_RESULT_SKIP) {
289                                 transientTestcaseFailures.add(testCaseName);
290                             }
291                         }
292                     }
293 
294                     // Record test case breakages
295                     if (result != TestCaseResult.TEST_CASE_RESULT_PASS
296                             && result != TestCaseResult.TEST_CASE_RESULT_SKIP) {
297                         testCaseBreakageMap.put(testCaseName, testCase);
298                     }
299                 }
300             }
301         }
302 
303         Set<String> buildIdList = new HashSet<>();
304         List<DeviceInfoEntity> devices = new ArrayList<>();
305         Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.getKey());
306         for (Entity device : datastore.prepare(deviceQuery).asIterable()) {
307             DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device);
308             if (deviceEntity == null) {
309                 continue;
310             }
311             buildIdList.add(deviceEntity.getBuildId());
312             devices.add(deviceEntity);
313         }
314         String footer = EmailHelper.getEmailFooter(mostRecentRun, devices, link);
315         String buildId = StringUtils.join(buildIdList, ",");
316 
317         for (String testCaseName : mostRecentTestCaseResults.keySet()) {
318             TestCaseResult mostRecentResult = mostRecentTestCaseResults.get(testCaseName);
319             boolean previouslyFailed = failedTestCaseMap.containsKey(testCaseName);
320             if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_SKIP) {
321                 // persist previous status
322                 if (previouslyFailed) {
323                     skippedTestcaseFailures.add(testCaseName);
324                     failingTestCases.add(
325                             new TestCaseReference(failedTestCaseMap.get(testCaseName)));
326                 } else {
327                     ++passingTestcaseCount;
328                 }
329             } else if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_PASS) {
330                 ++passingTestcaseCount;
331                 if (previouslyFailed && !transientTestcaseFailures.contains(testCaseName)) {
332                     fixedTestcases.add(testCaseName);
333                 }
334             } else {
335                 if (!previouslyFailed) {
336                     newTestcaseFailures.add(testCaseName);
337                     failingTestCases.add(
338                             new TestCaseReference(testCaseBreakageMap.get(testCaseName)));
339                 } else {
340                     continuedTestcaseFailures.add(testCaseName);
341                     failingTestCases.add(
342                             new TestCaseReference(failedTestCaseMap.get(testCaseName)));
343                 }
344             }
345         }
346 
347         Set<String> acknowledgedFailures =
348                 separateAcknowledged(newTestcaseFailures, devices, testAcks);
349         acknowledgedFailures.addAll(
350                 separateAcknowledged(transientTestcaseFailures, devices, testAcks));
351         acknowledgedFailures.addAll(
352                 separateAcknowledged(continuedTestcaseFailures, devices, testAcks));
353 
354         String summary = new String();
355         if (newTestcaseFailures.size() + continuedTestcaseFailures.size() > 0) {
356             summary += "The following test cases failed in the latest test run:<br>";
357 
358             // Add new test case failures to top of summary in bold font.
359             List<String> sortedNewTestcaseFailures = new ArrayList<>(newTestcaseFailures);
360             sortedNewTestcaseFailures.sort(Comparator.naturalOrder());
361             for (String testcaseName : sortedNewTestcaseFailures) {
362                 summary += "- " + "<b>" + testcaseName + "</b><br>";
363             }
364 
365             // Add continued test case failures to summary.
366             List<String> sortedContinuedTestcaseFailures =
367                     new ArrayList<>(continuedTestcaseFailures);
368             sortedContinuedTestcaseFailures.sort(Comparator.naturalOrder());
369             for (String testcaseName : sortedContinuedTestcaseFailures) {
370                 summary += "- " + testcaseName + "<br>";
371             }
372         }
373         if (fixedTestcases.size() > 0) {
374             // Add fixed test cases to summary.
375             summary += "<br><br>The following test cases were fixed in the latest test run:<br>";
376             List<String> sortedFixedTestcases = new ArrayList<>(fixedTestcases);
377             sortedFixedTestcases.sort(Comparator.naturalOrder());
378             for (String testcaseName : sortedFixedTestcases) {
379                 summary += "- <i>" + testcaseName + "</i><br>";
380             }
381         }
382         if (transientTestcaseFailures.size() > 0) {
383             // Add transient test case failures to summary.
384             summary += "<br><br>The following transient test case failures occured:<br>";
385             List<String> sortedTransientTestcaseFailures =
386                     new ArrayList<>(transientTestcaseFailures);
387             sortedTransientTestcaseFailures.sort(Comparator.naturalOrder());
388             for (String testcaseName : sortedTransientTestcaseFailures) {
389                 summary += "- " + testcaseName + "<br>";
390             }
391         }
392         if (skippedTestcaseFailures.size() > 0) {
393             // Add skipped test case failures to summary.
394             summary += "<br><br>The following test cases have not been run since failing:<br>";
395             List<String> sortedSkippedTestcaseFailures = new ArrayList<>(skippedTestcaseFailures);
396             sortedSkippedTestcaseFailures.sort(Comparator.naturalOrder());
397             for (String testcaseName : sortedSkippedTestcaseFailures) {
398                 summary += "- " + testcaseName + "<br>";
399             }
400         }
401         if (acknowledgedFailures.size() > 0) {
402             // Add acknowledged test case failures to summary.
403             List<String> sortedAcknowledgedFailures = new ArrayList<>(acknowledgedFailures);
404             sortedAcknowledgedFailures.sort(Comparator.naturalOrder());
405             if (acknowledgedFailures.size() > 0) {
406                 summary +=
407                         "<br><br>The following acknowledged test case failures continued to fail:<br>";
408                 for (String testcaseName : sortedAcknowledgedFailures) {
409                     summary += "- " + testcaseName + "<br>";
410                 }
411             }
412         }
413 
414         String testName = mostRecentRun.getKey().getParent().getName();
415         String uploadDateString = TimeUtil.getDateString(mostRecentRun.getStartTimestamp());
416         String subject = "VTS Test Alert: " + testName + " @ " + uploadDateString;
417         if (newTestcaseFailures.size() > 0) {
418             String body =
419                     "Hello,<br><br>New test case failure(s) in "
420                             + testName
421                             + " for device build ID(s): "
422                             + buildId
423                             + ".<br><br>"
424                             + summary
425                             + footer;
426             try {
427                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
428             } catch (MessagingException | UnsupportedEncodingException e) {
429                 logger.log(Level.WARNING, "Error composing email : ", e);
430             }
431         } else if (continuedTestcaseFailures.size() > 0) {
432             String body =
433                     "Hello,<br><br>Continuous test case failure(s) in "
434                             + testName
435                             + " for device build ID(s): "
436                             + buildId
437                             + ".<br><br>"
438                             + summary
439                             + footer;
440             try {
441                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
442             } catch (MessagingException | UnsupportedEncodingException e) {
443                 logger.log(Level.WARNING, "Error composing email : ", e);
444             }
445         } else if (transientTestcaseFailures.size() > 0) {
446             String body =
447                     "Hello,<br><br>Transient test case failure(s) in "
448                             + testName
449                             + " but tests all "
450                             + "are passing in the latest device build(s): "
451                             + buildId
452                             + ".<br><br>"
453                             + summary
454                             + footer;
455             try {
456                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
457             } catch (MessagingException | UnsupportedEncodingException e) {
458                 logger.log(Level.WARNING, "Error composing email : ", e);
459             }
460         } else if (fixedTestcases.size() > 0) {
461             String body =
462                     "Hello,<br><br>All test cases passed in "
463                             + testName
464                             + " for device build ID(s): "
465                             + buildId
466                             + "!<br><br>"
467                             + summary
468                             + footer;
469             try {
470                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
471             } catch (MessagingException | UnsupportedEncodingException e) {
472                 logger.log(Level.WARNING, "Error composing email : ", e);
473             }
474         }
475         return new TestStatusEntity(
476                 testName,
477                 mostRecentRun.getStartTimestamp(),
478                 passingTestcaseCount,
479                 failingTestCases.size(),
480                 failingTestCases);
481     }
482 
483     /**
484      * Add a task to process test run data
485      *
486      * @param testRunKey The key of the test run whose data process.
487      */
addTask(Key testRunKey)488     public static void addTask(Key testRunKey) {
489         Queue queue = QueueFactory.getDefaultQueue();
490         String keyString = KeyFactory.keyToString(testRunKey);
491         queue.add(
492                 TaskOptions.Builder.withUrl(ALERT_JOB_URL)
493                         .param("runKey", keyString)
494                         .method(TaskOptions.Method.POST));
495     }
496 
497     @Override
doPost(HttpServletRequest request, HttpServletResponse response)498     public void doPost(HttpServletRequest request, HttpServletResponse response)
499             throws IOException {
500         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
501         String runKeyString = request.getParameter("runKey");
502 
503         Key testRunKey;
504         try {
505             testRunKey = KeyFactory.stringToKey(runKeyString);
506         } catch (IllegalArgumentException e) {
507             logger.log(Level.WARNING, "Invalid key specified: " + runKeyString);
508             return;
509         }
510         String testName = testRunKey.getParent().getName();
511 
512         TestStatusEntity status = null;
513         Key statusKey = KeyFactory.createKey(TestStatusEntity.KIND, testName);
514         try {
515             status = TestStatusEntity.fromEntity(datastore.get(statusKey));
516         } catch (EntityNotFoundException e) {
517             // no existing status
518         }
519         if (status == null) {
520             status = new TestStatusEntity(testName);
521         }
522         if (status.getUpdatedTimestamp() >= testRunKey.getId()) {
523             // Another job has already updated the status first
524             return;
525         }
526         List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent());
527 
528         StringBuffer fullUrl = request.getRequestURL();
529         String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI()));
530         String link =
531                 baseUrl + "/show_tree?testName=" + testName + "&endTime=" + testRunKey.getId();
532 
533         List<Message> messageQueue = new ArrayList<>();
534         Map<String, TestCase> failedTestcaseMap = getCurrentFailures(status);
535         List<TestAcknowledgmentEntity> testAcks =
536                 getTestCaseAcknowledgments(testRunKey.getParent());
537         List<TestRunEntity> testRuns =
538                 getTestRuns(
539                         testRunKey.getParent(), status.getUpdatedTimestamp(), testRunKey.getId());
540         if (testRuns.size() == 0) return;
541 
542         TestStatusEntity newStatus =
543                 getTestStatus(testRuns, link, failedTestcaseMap, testAcks, emails, messageQueue);
544         if (newStatus == null) {
545             // No changes to status
546             return;
547         }
548 
549         int retries = 0;
550         while (true) {
551             Transaction txn = datastore.beginTransaction();
552             try {
553                 try {
554                     status = TestStatusEntity.fromEntity(datastore.get(statusKey));
555                 } catch (EntityNotFoundException e) {
556                     // no status left
557                 }
558                 if (status == null
559                         || status.getUpdatedTimestamp() >= newStatus.getUpdatedTimestamp()) {
560                     txn.rollback();
561                 } else { // This update is most recent.
562                     datastore.put(newStatus.toEntity());
563                     txn.commit();
564                     EmailHelper.sendAll(messageQueue);
565                 }
566                 break;
567             } catch (ConcurrentModificationException
568                     | DatastoreFailureException
569                     | DatastoreTimeoutException e) {
570                 logger.log(Level.WARNING, "Retrying alert job insert: " + statusKey);
571                 if (retries++ >= DatastoreHelper.MAX_WRITE_RETRIES) {
572                     logger.log(Level.SEVERE, "Exceeded alert job retries: " + statusKey);
573                     throw e;
574                 }
575             } finally {
576                 if (txn.isActive()) {
577                     txn.rollback();
578                 }
579             }
580         }
581     }
582 }
583