1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef TEST_TESTSUPPORT_PERF_RESULT_REPORTER_H_
12 #define TEST_TESTSUPPORT_PERF_RESULT_REPORTER_H_
13 
14 #include <string>
15 #include <unordered_map>
16 
17 #include "absl/types/optional.h"
18 #include "api/array_view.h"
19 #include "test/testsupport/perf_test.h"
20 
21 namespace webrtc {
22 namespace test {
23 
24 // These match the units in histogram.proto (in third_party/catapult).
25 enum class Unit {
26   kMs,
27   kMsBestFitFormat,
28   kMsTs,
29   kNPercent,
30   kSizeInBytes,
31   kBytesPerSecond,
32   kHertz,
33   kUnitless,
34   kCount,
35   kSigma,
36 };
37 
38 struct MetricInfo {
39   Unit unit;
40   ImproveDirection improve_direction;
41 };
42 
43 // A helper class for using the perf test printing functions safely, as
44 // otherwise it's easy to accidentally mix up arguments to produce usable but
45 // malformed perf data. See https://crbug.com/923564.
46 //
47 // Sample usage:
48 // auto reporter = PerfResultReporter("ramp_up_time", "bwe_15s");
49 // reporter.RegisterImportantMetric(
50 //     "_turn_over_tcp", Unit::kMs, ImproveDirection::kBiggerIsBetter);
51 // reporter.RegisterImportantMetric("_cpu_time", Unit::kMs);
52 // ...
53 // reporter.AddResult("turn_over_tcp", GetTurnOverTcpTime());
54 // reporter.AddResult("turn_over_udp", GetTurnOverUdpTime());
55 //
56 // This will show in the dashboard as
57 // (test binary name) > (bot) > ramp_up_time_turn_over_tcp > bwe_15s.
58 // (test binary name) > (bot) > ramp_up_time_turn_over_udp > bwe_15s.
59 //
60 // If you add more reporters that cover other user stories, they will show up
61 // as separate subtests (e.g. next to bwe_15s).
62 class PerfResultReporter {
63  public:
64   PerfResultReporter(const std::string& metric_basename,
65                      const std::string& story_name);
66   ~PerfResultReporter();
67 
68   void RegisterMetric(const std::string& metric_suffix, Unit unit);
69   void RegisterMetric(const std::string& metric_suffix,
70                       Unit unit,
71                       ImproveDirection improve_direction);
72   void AddResult(const std::string& metric_suffix, size_t value) const;
73   void AddResult(const std::string& metric_suffix, double value) const;
74 
75   void AddResultList(const std::string& metric_suffix,
76                      rtc::ArrayView<const double> values) const;
77 
78   // Users should prefer AddResultList if possible, as otherwise the min/max
79   // values reported on the perf dashboard aren't useful.
80   // |mean_and_error| should be a comma-separated string of mean then
81   // error/stddev, e.g. "2.4,0.5".
82   void AddResultMeanAndError(const std::string& metric_suffix,
83                              const double mean,
84                              const double error);
85 
86   // Returns the metric info if it has been registered.
87   absl::optional<MetricInfo> GetMetricInfo(
88       const std::string& metric_suffix) const;
89 
90  private:
91   MetricInfo GetMetricInfoOrFail(const std::string& metric_suffix) const;
92 
93   std::string metric_basename_;
94   std::string story_name_;
95   std::unordered_map<std::string, MetricInfo> metric_map_;
96 };
97 
98 }  // namespace test
99 }  // namespace webrtc
100 
101 #endif  // TEST_TESTSUPPORT_PERF_RESULT_REPORTER_H_
102