1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""The datastore models for graph data.
6
7The Chromium project uses Buildbot to run its performance tests, and the
8structure of the data for the Performance Dashboard reflects this. Metadata
9about tests are structured in a hierarchy of Master, Bot, and Test entities.
10Master and Bot entities represent Buildbot masters and builders respectively,
11and Test entities represent groups of results, or individual data series.
12
13For example, entities might be structured as follows:
14
15  Master: ChromiumPerf
16    Bot: win7
17      Test: page_cycler.moz
18        Test: times
19          Test: page_load_time
20          Test: page_load_time_ref
21          Test: www.amazon.com
22          Test: www.bing.com
23        Test: commit_charge
24          Test: ref
25          Test: www.amazon.com
26          Test: www.bing.com
27
28The graph data points are represented by Row entities. Each Row entity contains
29a revision and value, which are its X and Y values on a graph, and any other
30metadata associated with an individual performance test result.
31
32The keys of the Row entities for a particular data series are start with a
33TestContainer key, instead of a Test key. This way, the Row entities for each
34data series are in a different "entity group". This allows a faster rate of
35putting data in the datastore for many series at once.
36
37For example, Row entities are organized like this:
38
39  TestContainer: ChromiumPerf/win7/page_cycler.moz/times/page_load_time
40    Row: revision 12345, value 2.5
41    Row: revision 12346, value 2.0
42    Row: revision 12347, value 2.1
43  TestContainer: ChromiumPerf/win7/page_cycler.moz/times/page_load_time_ref
44    Row: revision 12345, value 2.4
45    Row: revision 12346, value 2.0
46    Row: revision 12347, value 2.2
47  TestContainer: ChromiumPerf/win7/page_cycler.moz/commit_charge
48    Row: revision 12345, value 10
49    Row: revision 12346, value 12
50    Row: revision 12347, value 11
51
52
53IMPORTANT: If you add new kinds to this file, you must also add them to the
54Daily Backup url in cron.yaml in order for them to be properly backed up.
55See: https://developers.google.com/appengine/articles/scheduled_backups
56"""
57
58import logging
59
60from google.appengine.ext import ndb
61
62from dashboard import layered_cache
63from dashboard import utils
64from dashboard.models import anomaly
65from dashboard.models import anomaly_config
66from dashboard.models import internal_only_model
67from dashboard.models import sheriff as sheriff_module
68from dashboard.models import stoppage_alert as stoppage_alert_module
69
70# Maximum level of nested tests.
71MAX_TEST_ANCESTORS = 10
72
73# Keys to the datastore-based cache. See stored_object.
74LIST_TESTS_SUBTEST_CACHE_KEY = 'list_tests_get_tests_new_%s_%s_%s'
75
76_MAX_STRING_LENGTH = 500
77
78
79class Master(internal_only_model.InternalOnlyModel):
80  """Information about the Buildbot master.
81
82  Masters are keyed by name, e.g. 'ChromiumGPU' or 'ChromiumPerf'.
83  All Bot entities that are Buildbot slaves of one master are children of one
84  Master entity in the datastore.
85  """
86  # Master has no properties; the name of the master is the ID.
87
88
89class Bot(internal_only_model.InternalOnlyModel):
90  """Information about a Buildbot slave that runs perf tests.
91
92  Bots are keyed by name, e.g. 'xp-release-dual-core'. A Bot entity contains
93  information about whether the tests are only viewable to internal users, and
94  each bot has a parent that is a Master entity. A Bot is be the ancestor of
95  the Test entities that run on it.
96  """
97  internal_only = ndb.BooleanProperty(default=False, indexed=True)
98
99
100class Test(internal_only_model.CreateHookInternalOnlyModel):
101  """A Test entity is a node in a hierarchy of tests.
102
103  A Test entity can represent a specific series of results which will be
104  plotted on a graph, or it can represent a group of such series of results, or
105  both. A Test entity that the property has_rows set to True corresponds to a
106  trace on a graph, and the parent Test for a group of such tests corresponds to
107  a graph with several traces. A parent Test for that test would correspond to a
108  group of related graphs. Top-level Tests (also known as test suites) are
109  parented by a Bot.
110
111  Tests are keyed by name, and they also contain other metadata such as
112  description and units.
113
114  NOTE: If you remove any properties from Test, they should be added to the
115  TEST_EXCLUDE_PROPERTIES list in migrate_test_names.py.
116  """
117  internal_only = ndb.BooleanProperty(default=False, indexed=True)
118
119  # Sheriff rotation for this test. Rotations are specified by regular
120  # expressions that can be edited at /edit_sheriffs.
121  sheriff = ndb.KeyProperty(kind=sheriff_module.Sheriff, indexed=True)
122
123  # There is a default anomaly threshold config (in anomaly.py), and it can
124  # be overridden for a group of tests by using /edit_sheriffs.
125  overridden_anomaly_config = ndb.KeyProperty(
126      kind=anomaly_config.AnomalyConfig, indexed=True)
127
128  # Keep track of what direction is an improvement for this graph so we can
129  # filter out alerts on regressions.
130  improvement_direction = ndb.IntegerProperty(
131      default=anomaly.UNKNOWN,
132      choices=[
133          anomaly.UP,
134          anomaly.DOWN,
135          anomaly.UNKNOWN
136      ],
137      indexed=False
138  )
139
140  # Units of the child Rows of this Test, or None if there are no child Rows.
141  units = ndb.StringProperty(indexed=False)
142
143  # The last alerted revision is used to avoid duplicate alerts.
144  last_alerted_revision = ndb.IntegerProperty(indexed=False)
145
146  # Whether or not the test has child rows. Set by hook on Row class put.
147  has_rows = ndb.BooleanProperty(default=False, indexed=True)
148
149  # If there is a currently a StoppageAlert that indicates that data hasn't
150  # been received for some time, then will be set. Otherwise, it is None.
151  stoppage_alert = ndb.KeyProperty(
152      kind=stoppage_alert_module.StoppageAlert, indexed=True)
153
154  # A test is marked "deprecated" if no new points have been received for
155  # a long time; these tests should usually not be listed.
156  deprecated = ndb.BooleanProperty(default=False, indexed=True)
157
158  # For top-level test entities, this is a list of sub-tests that are checked
159  # for alerts (i.e. they have a sheriff). For other tests, this is empty.
160  monitored = ndb.KeyProperty(repeated=True, indexed=True)
161
162  # Description of what the test measures.
163  description = ndb.TextProperty(indexed=True)
164
165  # Source code location of the test. Optional.
166  code = ndb.StringProperty(indexed=False, repeated=True)
167
168  # Command to run the test. Optional.
169  command_line = ndb.StringProperty(indexed=False)
170
171  # Computed properties are treated like member variables, so they have
172  # lowercase names, even though they look like methods to pylint.
173  # pylint: disable=invalid-name
174
175  @ndb.ComputedProperty
176  def bot(self):  # pylint: disable=invalid-name
177    """Immediate parent Bot entity, or None if this is not a test suite."""
178    parent = self.key.parent()
179    if parent.kind() == 'Bot':
180      return parent
181    return None
182
183  @ndb.ComputedProperty
184  def parent_test(self):  # pylint: disable=invalid-name
185    """Immediate parent Test entity, or None if this is a test suite."""
186    parent = self.key.parent()
187    if parent.kind() == 'Test':
188      return parent
189    return None
190
191  @property
192  def test_path(self):
193    """Slash-separated list of key parts, 'master/bot/suite/chart/...'."""
194    return utils.TestPath(self.key)
195
196  @ndb.ComputedProperty
197  def master_name(self):
198    return self.key.pairs()[0][1]
199
200  @ndb.ComputedProperty
201  def bot_name(self):
202    return self.key.pairs()[1][1]
203
204  @ndb.ComputedProperty
205  def suite_name(self):
206    return self.key.pairs()[2][1]
207
208  @ndb.ComputedProperty
209  def test_part1_name(self):
210    pairs = self.key.pairs()
211    if len(pairs) < 4:
212      return ''
213    return self.key.pairs()[3][1]
214
215  @ndb.ComputedProperty
216  def test_part2_name(self):
217    pairs = self.key.pairs()
218    if len(pairs) < 5:
219      return ''
220    return self.key.pairs()[4][1]
221
222  @ndb.ComputedProperty
223  def test_part3_name(self):
224    pairs = self.key.pairs()
225    if len(pairs) < 6:
226      return ''
227    return self.key.pairs()[5][1]
228
229  @ndb.ComputedProperty
230  def test_part4_name(self):
231    pairs = self.key.pairs()
232    if len(pairs) < 7:
233      return ''
234    return self.key.pairs()[6][0]
235
236  @classmethod
237  def _GetMasterBotSuite(cls, key):
238    while key and key.parent():
239      if key.parent().kind() == 'Bot':
240        if not key.parent().parent():
241          return None
242        return (key.parent().parent().string_id(),
243                key.parent().string_id(),
244                key.string_id())
245      key = key.parent()
246    return None
247
248  def __init__(self, *args, **kwargs):
249    # Indexed StringProperty has a maximum length. If this length is exceeded,
250    # then an error will be thrown in ndb.Model.__init__.
251    # Truncate the "description" property if necessary.
252    description = kwargs.get('description') or ''
253    kwargs['description'] = description[:_MAX_STRING_LENGTH]
254    super(Test, self).__init__(*args, **kwargs)
255
256  def _pre_put_hook(self):
257    """This method is called before a Test is put into the datastore.
258
259    Here, we check the sheriffs and anomaly configs to make sure they are
260    current. We also update the monitored list of the test suite.
261    """
262    # Set the sheriff to the first sheriff (alphabetically by sheriff name)
263    # that has a test pattern that matches this test.
264    self.sheriff = None
265    for sheriff_entity in sheriff_module.Sheriff.query().fetch():
266      for pattern in sheriff_entity.patterns:
267        if utils.TestMatchesPattern(self, pattern):
268          self.sheriff = sheriff_entity.key
269      if self.sheriff:
270        break
271
272    # If this Test is monitored, add it to the monitored list of its test suite.
273    # A test is be monitored iff it has a sheriff, and monitored tests are
274    # tracked in the monitored list of a test suite Test entity.
275    test_suite = ndb.Key(*self.key.flat()[:6]).get()
276    if self.sheriff:
277      if test_suite and self.key not in test_suite.monitored:
278        test_suite.monitored.append(self.key)
279        test_suite.put()
280    elif test_suite and self.key in test_suite.monitored:
281      test_suite.monitored.remove(self.key)
282      test_suite.put()
283
284    # Set the anomaly threshold config to the first one that has a test pattern
285    # that matches this test, if there is one. Anomaly configs are sorted by
286    # name, so that a config with a name that comes earlier lexicographically
287    # is considered higher-priority.
288    self.overridden_anomaly_config = None
289    anomaly_configs = anomaly_config.AnomalyConfig.query().fetch()
290    anomaly_configs.sort(key=lambda config: config.key.string_id())
291    for anomaly_config_entity in anomaly_configs:
292      for pattern in anomaly_config_entity.patterns:
293        if utils.TestMatchesPattern(self, pattern):
294          self.overridden_anomaly_config = anomaly_config_entity.key
295      if self.overridden_anomaly_config:
296        break
297
298  def CreateCallback(self):
299    """Called when the entity is first saved."""
300    if self.key.parent().kind() != 'Bot':
301      layered_cache.Delete(
302          LIST_TESTS_SUBTEST_CACHE_KEY % self._GetMasterBotSuite(self.key))
303
304  @classmethod
305  # pylint: disable=unused-argument
306  def _pre_delete_hook(cls, key):
307    if key.parent() and key.parent().kind() != 'Bot':
308      layered_cache.Delete(
309          LIST_TESTS_SUBTEST_CACHE_KEY % Test._GetMasterBotSuite(key))
310
311
312class LastAddedRevision(ndb.Model):
313  """Represents the last added revision for a test path.
314
315  The reason this property is separated from Test entity is to avoid contention
316  issues (Frequent update of entity within the same group).  This property is
317  updated very frequent in /add_point.
318  """
319  revision = ndb.IntegerProperty(indexed=False)
320
321
322class Row(internal_only_model.InternalOnlyModel, ndb.Expando):
323  """A Row represents one data point.
324
325  A Row has a revision and a value, which are the X and Y values, respectively.
326  Each Row belongs to one Test, along with all of the other Row entities that
327  it is plotted with. Rows are keyed by revision.
328
329  In addition to the properties defined below, Row entities may also have other
330  properties which specify additional supplemental data. These are called
331  "supplemental columns", and should have the following prefixes:
332    d_: A data point, such as d_1st_run or d_50th_percentile. FloatProperty.
333    r_: Revision such as r_webkit or r_v8. StringProperty, limited to 25
334        characters, '0-9' and '.'.
335    a_: Annotation such as a_chrome_bugid or a_gasp_anomaly. StringProperty.
336  """
337  # Don't index by default (only explicitly indexed properties are indexed).
338  _default_indexed = False
339  internal_only = ndb.BooleanProperty(default=False, indexed=True)
340
341  # The parent_test is the key of the Test entity that this Row belongs to.
342  @ndb.ComputedProperty
343  def parent_test(self):  # pylint: disable=invalid-name
344    # The Test entity that a Row belongs to isn't actually its parent in the
345    # datastore. Rather, the parent key of each Row contains a test path, which
346    # contains the information necessary to get the actual Test key.
347    return utils.TestKey(self.key.parent().string_id())
348
349  # Points in each graph are sorted by "revision". This is usually a Chromium
350  # SVN version number, but it might also be any other integer, as long as
351  # newer points have higher numbers.
352  @ndb.ComputedProperty
353  def revision(self):  # pylint: disable=invalid-name
354    return self.key.integer_id()
355
356  # The time the revision was added to the dashboard is tracked in order
357  # to too many points from being added in a short period of time, which would
358  # indicate an error or malicious code.
359  timestamp = ndb.DateTimeProperty(auto_now_add=True, indexed=True)
360
361  # The Y-value at this point.
362  value = ndb.FloatProperty(indexed=True)
363
364  # The standard deviation at this point. Optional.
365  error = ndb.FloatProperty(indexed=False)
366
367  def _pre_put_hook(self):
368    """Sets the has_rows property of the parent test before putting this Row.
369
370    This isn't atomic because the parent_test put() and Row put() don't happen
371    in the same transaction. But in practice it shouldn't be an issue because
372    the parent test will get more points as the test runs.
373    """
374    parent_test = self.parent_test.get()
375
376    # If the Test pointed to by parent_test is not valid, that indicates
377    # that a Test entity was not properly created in add_point.
378    if not parent_test:
379      parent_key = self.key.parent()
380      logging.warning('Row put without valid Test. Parent key: %s', parent_key)
381      return
382
383    if not parent_test.has_rows:
384      parent_test.has_rows = True
385      parent_test.put()
386