1// Protocol messages for describing the results of benchmarks and unit tests.
2syntax = "proto3";
3
4package tensorflow;
5
6import "google/protobuf/any.proto";
7import "google/protobuf/wrappers.proto";
8
9option cc_enable_arenas = true;
10option java_outer_classname = "TestLogProtos";
11option java_multiple_files = true;
12option java_package = "org.tensorflow.util.testlog";
13
14message EntryValue {
15  oneof kind {
16    double double_value = 1;
17    string string_value = 2;
18  }
19}
20
21message MetricEntry {
22  // Metric name
23  string name = 1;
24
25  // Metric value
26  double value = 2;
27
28  // The minimum acceptable value for the metric if specified
29  google.protobuf.DoubleValue min_value = 3;
30
31  // The maximum acceptable value for the metric if specified
32  google.protobuf.DoubleValue max_value = 4;
33}
34
35// Each unit test or benchmark in a test or benchmark run provides
36// some set of information.  Here we provide some reasonable keys
37// one would expect to see, with optional key/value pairs for things
38// we haven't considered.
39//
40// This BenchmarkEntry should be emitted by each unit test or benchmark
41// reporter.
42message BenchmarkEntry {
43  // The name of the specific benchmark or test
44  // (e.g. BM_AdjustContrast_gpu_B_W_H)
45  string name = 1;
46
47  // If a benchmark, how many iterations it was run for
48  int64 iters = 2;
49
50  // Total cpu time used for all iterations (in seconds)
51  double cpu_time = 3;
52
53  // Total wall time used for all iterations (in seconds)
54  double wall_time = 4;
55
56  // Throughput (in MB/s)
57  double throughput = 5;
58
59  // Generic map from result key to value.
60  map<string, EntryValue> extras = 6;
61
62  // Metric name, value and expected range. This can include accuracy metrics
63  // typically used to determine whether the accuracy test has passed
64  repeated MetricEntry metrics = 7;
65}
66
67message BenchmarkEntries {
68  repeated BenchmarkEntry entry = 1;
69}
70
71message BuildConfiguration {
72  string mode = 1;               // opt, dbg, etc
73  repeated string cc_flags = 2;  // CC compiler flags, if known
74  repeated string opts = 3;      // Bazel compilation options, if known
75}
76
77message CommitId {
78  oneof kind {
79    // Submitted changelist.
80    int64 changelist = 1;
81    string hash = 2;
82  }
83  // Hash of intermediate change between hash/changelist and what was tested.
84  // Not used if the build is from a commit without modifications.
85  string snapshot = 3;
86  // Changelist tested if the change list is not already submitted.
87  int64 pending_changelist = 4;
88}
89
90message CPUInfo {
91  int64 num_cores = 1;
92
93  int64 num_cores_allowed = 2;
94
95  // How fast are these cpus?
96  double mhz_per_cpu = 3;
97
98  // Additional cpu information. For example,
99  // Intel Ivybridge with HyperThreading (24 cores) dL1:32KB dL2:256KB dL3:30MB
100  string cpu_info = 4;
101
102  // What kind of cpu scaling is enabled on the host.
103  // Examples include "performance", "ondemand", "conservative", "mixed".
104  string cpu_governor = 5;
105
106  // Cache sizes (in bytes), e.g. "L2": 262144 (for 256KB)
107  map<string, int64> cache_size = 6;
108}
109
110message MemoryInfo {
111  int64 total = 1;      // Total virtual memory in bytes
112  int64 available = 2;  // Immediately available memory in bytes
113}
114
115message GPUInfo {
116  string model = 1;   // e.g. "Tesla K40c"
117  string uuid = 2;    // Final entry in output of "nvidia-smi -L"
118  string bus_id = 3;  // e.g. "0000:04:00.0"
119}
120
121message PlatformInfo {
122  string bits = 1;     // e.g. '64bit'
123  string linkage = 2;  // e.g. 'ELF'
124  string machine = 3;  // e.g. 'i386'
125  string release = 4;  // e.g. '3.13.0-76-generic'
126  string system = 5;   // e.g. 'Linux'
127  string version = 6;  // e.g. '#120-Ubuntu SMP Mon Jan 18 15:59:10 UTC 2016'
128}
129
130message AvailableDeviceInfo {       // Matches DeviceAttributes
131  string name = 1;                  // Device name.
132  string type = 2;                  // Device type, e.g. 'CPU' or 'GPU'.
133  int64 memory_limit = 3;           // Memory capacity in bytes.
134  string physical_description = 4;  // The physical description of this device.
135}
136
137message MachineConfiguration {
138  // Host name of machine that ran the benchmark.
139  string hostname = 1;
140
141  // Unique serial number of the machine.
142  string serial_identifier = 7;
143
144  // Additional platform information.
145  PlatformInfo platform_info = 2;
146
147  // CPU Information.
148  CPUInfo cpu_info = 3;
149
150  // Other devices that are attached and relevant (e.g. GPUInfo).
151  repeated google.protobuf.Any device_info = 4;
152
153  // Devices accessible to the test (e.g. as given by list_local_devices).
154  repeated AvailableDeviceInfo available_device_info = 5;
155
156  MemoryInfo memory_info = 6;
157}
158
159// Run-specific items such as arguments to the test / benchmark.
160message RunConfiguration {
161  repeated string argument = 1;
162  // Environment variables used to run the test/benchmark.
163  map<string, string> env_vars = 2;
164}
165
166// The output of one benchmark / test run.  Each run contains a list of
167// tests or benchmarks, stored as BenchmarkEntry messages.
168//
169// This message should be emitted by the reporter (which runs the
170// test / BM in a subprocess and then reads the emitted BenchmarkEntry messages;
171// usually from a serialized json file, finally collecting them along
172// with additional information about the test run.
173message TestResults {
174  // The target of the run, e.g.:
175  //  //tensorflow/core:kernels_adjust_contrast_op_benchmark_test
176  string target = 1;
177
178  // The list of tests or benchmarks in this run.
179  BenchmarkEntries entries = 2;
180
181  // The configuration of the build (compiled opt? with cuda? any copts?)
182  BuildConfiguration build_configuration = 3;
183
184  // The commit id (git hash or changelist)
185  CommitId commit_id = 4;
186
187  // The time the run started (in seconds of UTC time since Unix epoch)
188  int64 start_time = 5;
189
190  // The amount of time the total run took (wall time in seconds)
191  double run_time = 6;
192
193  // Machine-specific parameters (Platform and CPU info)
194  MachineConfiguration machine_configuration = 7;
195
196  // Run-specific parameters (arguments, etc)
197  RunConfiguration run_configuration = 8;
198
199  // Benchmark target identifier.
200  string name = 9;
201
202  // The type of benchmark.
203  enum BenchmarkType {
204    UNKNOWN = 0;  // Fallback for protos written before Type was introduced.
205    CPP_MICROBENCHMARK = 1;
206    PYTHON_BENCHMARK = 2;
207    ANDROID_BENCHMARK = 3;
208    EDGE_BENCHMARK = 4;
209    IOS_BENCHMARK = 5;
210  }
211  BenchmarkType benchmark_type = 10;
212
213  // Used for differentiating between continuous and debug builds.
214  // Must be one of:
215  // * cbuild: results from continuous build.
216  // * presubmit: results from oneshot requests.
217  // * culprit: results from culprit finder rerun.
218  string run_mode = 11;
219
220  // TensorFlow version this benchmark runs against.
221  // This can be either set to full version or just the major version.
222  string tf_version = 12;
223}
224