1
2 #undef NDEBUG
3 #include <utility>
4
5 #include "benchmark/benchmark.h"
6 #include "output_test.h"
7
8 // ========================================================================= //
9 // ---------------------- Testing Prologue Output -------------------------- //
10 // ========================================================================= //
11
12 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
13 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
14 {"^[-]+$", MR_Next}});
AddContextCases()15 static int AddContextCases() {
16 AddCases(TC_ConsoleErr,
17 {
18 {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default},
19 {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
20 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
21 });
22 AddCases(TC_JSONOut,
23 {{"^\\{", MR_Default},
24 {"\"context\":", MR_Next},
25 {"\"date\": \"", MR_Next},
26 {"\"host_name\":", MR_Next},
27 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
28 MR_Next},
29 {"\"num_cpus\": %int,$", MR_Next},
30 {"\"mhz_per_cpu\": %float,$", MR_Next},
31 {"\"cpu_scaling_enabled\": ", MR_Next},
32 {"\"caches\": \\[$", MR_Next}});
33 auto const& Info = benchmark::CPUInfo::Get();
34 auto const& Caches = Info.caches;
35 if (!Caches.empty()) {
36 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
37 }
38 for (size_t I = 0; I < Caches.size(); ++I) {
39 std::string num_caches_str =
40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
41 AddCases(
42 TC_ConsoleErr,
43 {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}});
44 AddCases(TC_JSONOut, {{"\\{$", MR_Next},
45 {"\"type\": \"", MR_Next},
46 {"\"level\": %int,$", MR_Next},
47 {"\"size\": %int,$", MR_Next},
48 {"\"num_sharing\": %int$", MR_Next},
49 {"}[,]{0,1}$", MR_Next}});
50 }
51 AddCases(TC_JSONOut, {{"],$"}});
52 auto const& LoadAvg = Info.load_avg;
53 if (!LoadAvg.empty()) {
54 AddCases(TC_ConsoleErr,
55 {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
56 }
57 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
58 return 0;
59 }
60 int dummy_register = AddContextCases();
61 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
62
63 // ========================================================================= //
64 // ------------------------ Testing Basic Output --------------------------- //
65 // ========================================================================= //
66
BM_basic(benchmark::State & state)67 void BM_basic(benchmark::State& state) {
68 for (auto _ : state) {
69 }
70 }
71 BENCHMARK(BM_basic);
72
73 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
74 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
75 {"\"run_name\": \"BM_basic\",$", MR_Next},
76 {"\"run_type\": \"iteration\",$", MR_Next},
77 {"\"iterations\": %int,$", MR_Next},
78 {"\"real_time\": %float,$", MR_Next},
79 {"\"cpu_time\": %float,$", MR_Next},
80 {"\"time_unit\": \"ns\"$", MR_Next},
81 {"}", MR_Next}});
82 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
83
84 // ========================================================================= //
85 // ------------------------ Testing Bytes per Second Output ---------------- //
86 // ========================================================================= //
87
BM_bytes_per_second(benchmark::State & state)88 void BM_bytes_per_second(benchmark::State& state) {
89 for (auto _ : state) {
90 }
91 state.SetBytesProcessed(1);
92 }
93 BENCHMARK(BM_bytes_per_second);
94
95 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
96 "bytes_per_second=%float[kM]{0,1}/s$"}});
97 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
98 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
99 {"\"run_type\": \"iteration\",$", MR_Next},
100 {"\"iterations\": %int,$", MR_Next},
101 {"\"real_time\": %float,$", MR_Next},
102 {"\"cpu_time\": %float,$", MR_Next},
103 {"\"time_unit\": \"ns\",$", MR_Next},
104 {"\"bytes_per_second\": %float$", MR_Next},
105 {"}", MR_Next}});
106 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
107
108 // ========================================================================= //
109 // ------------------------ Testing Items per Second Output ---------------- //
110 // ========================================================================= //
111
BM_items_per_second(benchmark::State & state)112 void BM_items_per_second(benchmark::State& state) {
113 for (auto _ : state) {
114 }
115 state.SetItemsProcessed(1);
116 }
117 BENCHMARK(BM_items_per_second);
118
119 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
120 "items_per_second=%float[kM]{0,1}/s$"}});
121 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
122 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
123 {"\"run_type\": \"iteration\",$", MR_Next},
124 {"\"iterations\": %int,$", MR_Next},
125 {"\"real_time\": %float,$", MR_Next},
126 {"\"cpu_time\": %float,$", MR_Next},
127 {"\"time_unit\": \"ns\",$", MR_Next},
128 {"\"items_per_second\": %float$", MR_Next},
129 {"}", MR_Next}});
130 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
131
132 // ========================================================================= //
133 // ------------------------ Testing Label Output --------------------------- //
134 // ========================================================================= //
135
BM_label(benchmark::State & state)136 void BM_label(benchmark::State& state) {
137 for (auto _ : state) {
138 }
139 state.SetLabel("some label");
140 }
141 BENCHMARK(BM_label);
142
143 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
144 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
145 {"\"run_name\": \"BM_label\",$", MR_Next},
146 {"\"run_type\": \"iteration\",$", MR_Next},
147 {"\"iterations\": %int,$", MR_Next},
148 {"\"real_time\": %float,$", MR_Next},
149 {"\"cpu_time\": %float,$", MR_Next},
150 {"\"time_unit\": \"ns\",$", MR_Next},
151 {"\"label\": \"some label\"$", MR_Next},
152 {"}", MR_Next}});
153 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
154 "label\"%csv_label_report_end$"}});
155
156 // ========================================================================= //
157 // ------------------------ Testing Error Output --------------------------- //
158 // ========================================================================= //
159
BM_error(benchmark::State & state)160 void BM_error(benchmark::State& state) {
161 state.SkipWithError("message");
162 for (auto _ : state) {
163 }
164 }
165 BENCHMARK(BM_error);
166 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
167 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
168 {"\"run_name\": \"BM_error\",$", MR_Next},
169 {"\"run_type\": \"iteration\",$", MR_Next},
170 {"\"error_occurred\": true,$", MR_Next},
171 {"\"error_message\": \"message\",$", MR_Next}});
172
173 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
174
175 // ========================================================================= //
176 // ------------------------ Testing No Arg Name Output -----------------------
177 // //
178 // ========================================================================= //
179
BM_no_arg_name(benchmark::State & state)180 void BM_no_arg_name(benchmark::State& state) {
181 for (auto _ : state) {
182 }
183 }
184 BENCHMARK(BM_no_arg_name)->Arg(3);
185 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
186 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
187 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
188 {"\"run_type\": \"iteration\",$", MR_Next}});
189 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
190
191 // ========================================================================= //
192 // ------------------------ Testing Arg Name Output ----------------------- //
193 // ========================================================================= //
194
BM_arg_name(benchmark::State & state)195 void BM_arg_name(benchmark::State& state) {
196 for (auto _ : state) {
197 }
198 }
199 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
200 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
201 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
202 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
203 {"\"run_type\": \"iteration\",$", MR_Next}});
204 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
205
206 // ========================================================================= //
207 // ------------------------ Testing Arg Names Output ----------------------- //
208 // ========================================================================= //
209
BM_arg_names(benchmark::State & state)210 void BM_arg_names(benchmark::State& state) {
211 for (auto _ : state) {
212 }
213 }
214 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
215 ADD_CASES(TC_ConsoleOut,
216 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
217 ADD_CASES(TC_JSONOut,
218 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
219 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
220 {"\"run_type\": \"iteration\",$", MR_Next}});
221 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
222
223 // ========================================================================= //
224 // ------------------------ Testing Big Args Output ------------------------ //
225 // ========================================================================= //
226
BM_BigArgs(benchmark::State & state)227 void BM_BigArgs(benchmark::State& state) {
228 for (auto _ : state) {
229 }
230 }
231 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
232 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
233 {"^BM_BigArgs/2147483648 %console_report$"}});
234
235 // ========================================================================= //
236 // ----------------------- Testing Complexity Output ----------------------- //
237 // ========================================================================= //
238
BM_Complexity_O1(benchmark::State & state)239 void BM_Complexity_O1(benchmark::State& state) {
240 for (auto _ : state) {
241 }
242 state.SetComplexityN(state.range(0));
243 }
244 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
245 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
246 {"%RMS", "[ ]*[0-9]+ %"}});
247 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
248 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
249
250 // ========================================================================= //
251 // ----------------------- Testing Aggregate Output ------------------------ //
252 // ========================================================================= //
253
254 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)255 void BM_Repeat(benchmark::State& state) {
256 for (auto _ : state) {
257 }
258 }
259 // need two repetitions min to be able to output any aggregate output
260 BENCHMARK(BM_Repeat)->Repetitions(2);
261 ADD_CASES(TC_ConsoleOut,
262 {{"^BM_Repeat/repeats:2 %console_report$"},
263 {"^BM_Repeat/repeats:2 %console_report$"},
264 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
265 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
266 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
267 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
268 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
269 {"\"run_type\": \"iteration\",$", MR_Next},
270 {"\"name\": \"BM_Repeat/repeats:2\",$"},
271 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
272 {"\"run_type\": \"iteration\",$", MR_Next},
273 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
274 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
275 {"\"run_type\": \"aggregate\",$", MR_Next},
276 {"\"aggregate_name\": \"mean\",$", MR_Next},
277 {"\"iterations\": 2,$", MR_Next},
278 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
279 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
280 {"\"run_type\": \"aggregate\",$", MR_Next},
281 {"\"aggregate_name\": \"median\",$", MR_Next},
282 {"\"iterations\": 2,$", MR_Next},
283 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
284 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
285 {"\"run_type\": \"aggregate\",$", MR_Next},
286 {"\"aggregate_name\": \"stddev\",$", MR_Next},
287 {"\"iterations\": 2,$", MR_Next}});
288 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
289 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
290 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
291 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
292 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
293 // but for two repetitions, mean and median is the same, so let's repeat..
294 BENCHMARK(BM_Repeat)->Repetitions(3);
295 ADD_CASES(TC_ConsoleOut,
296 {{"^BM_Repeat/repeats:3 %console_report$"},
297 {"^BM_Repeat/repeats:3 %console_report$"},
298 {"^BM_Repeat/repeats:3 %console_report$"},
299 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
300 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
301 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
302 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
303 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
304 {"\"run_type\": \"iteration\",$", MR_Next},
305 {"\"name\": \"BM_Repeat/repeats:3\",$"},
306 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
307 {"\"run_type\": \"iteration\",$", MR_Next},
308 {"\"name\": \"BM_Repeat/repeats:3\",$"},
309 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
310 {"\"run_type\": \"iteration\",$", MR_Next},
311 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
312 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
313 {"\"run_type\": \"aggregate\",$", MR_Next},
314 {"\"aggregate_name\": \"mean\",$", MR_Next},
315 {"\"iterations\": 3,$", MR_Next},
316 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
317 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
318 {"\"run_type\": \"aggregate\",$", MR_Next},
319 {"\"aggregate_name\": \"median\",$", MR_Next},
320 {"\"iterations\": 3,$", MR_Next},
321 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
322 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
323 {"\"run_type\": \"aggregate\",$", MR_Next},
324 {"\"aggregate_name\": \"stddev\",$", MR_Next},
325 {"\"iterations\": 3,$", MR_Next}});
326 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
327 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
328 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
329 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
330 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
331 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
332 // median differs between even/odd number of repetitions, so just to be sure
333 BENCHMARK(BM_Repeat)->Repetitions(4);
334 ADD_CASES(TC_ConsoleOut,
335 {{"^BM_Repeat/repeats:4 %console_report$"},
336 {"^BM_Repeat/repeats:4 %console_report$"},
337 {"^BM_Repeat/repeats:4 %console_report$"},
338 {"^BM_Repeat/repeats:4 %console_report$"},
339 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
340 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
341 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
342 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
343 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
344 {"\"run_type\": \"iteration\",$", MR_Next},
345 {"\"name\": \"BM_Repeat/repeats:4\",$"},
346 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
347 {"\"run_type\": \"iteration\",$", MR_Next},
348 {"\"name\": \"BM_Repeat/repeats:4\",$"},
349 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
350 {"\"run_type\": \"iteration\",$", MR_Next},
351 {"\"name\": \"BM_Repeat/repeats:4\",$"},
352 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
353 {"\"run_type\": \"iteration\",$", MR_Next},
354 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
355 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
356 {"\"run_type\": \"aggregate\",$", MR_Next},
357 {"\"aggregate_name\": \"mean\",$", MR_Next},
358 {"\"iterations\": 4,$", MR_Next},
359 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
360 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
361 {"\"run_type\": \"aggregate\",$", MR_Next},
362 {"\"aggregate_name\": \"median\",$", MR_Next},
363 {"\"iterations\": 4,$", MR_Next},
364 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
365 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
366 {"\"run_type\": \"aggregate\",$", MR_Next},
367 {"\"aggregate_name\": \"stddev\",$", MR_Next},
368 {"\"iterations\": 4,$", MR_Next}});
369 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
370 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
371 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
372 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
373 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
374 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
375 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
376
377 // Test that a non-repeated test still prints non-aggregate results even when
378 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)379 void BM_RepeatOnce(benchmark::State& state) {
380 for (auto _ : state) {
381 }
382 }
383 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
384 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
385 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
386 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
387 {"\"run_type\": \"iteration\",$", MR_Next}});
388 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
389
390 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)391 void BM_SummaryRepeat(benchmark::State& state) {
392 for (auto _ : state) {
393 }
394 }
395 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
396 ADD_CASES(
397 TC_ConsoleOut,
398 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
399 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
400 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
401 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
402 ADD_CASES(TC_JSONOut,
403 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
404 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
405 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
406 {"\"run_type\": \"aggregate\",$", MR_Next},
407 {"\"aggregate_name\": \"mean\",$", MR_Next},
408 {"\"iterations\": 3,$", MR_Next},
409 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
410 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
411 {"\"run_type\": \"aggregate\",$", MR_Next},
412 {"\"aggregate_name\": \"median\",$", MR_Next},
413 {"\"iterations\": 3,$", MR_Next},
414 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
415 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
416 {"\"run_type\": \"aggregate\",$", MR_Next},
417 {"\"aggregate_name\": \"stddev\",$", MR_Next},
418 {"\"iterations\": 3,$", MR_Next}});
419 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
420 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
421 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
422 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
423
424 // Test that non-aggregate data is not displayed.
425 // NOTE: this test is kinda bad. we are only testing the display output.
426 // But we don't check that the file output still contains everything...
BM_SummaryDisplay(benchmark::State & state)427 void BM_SummaryDisplay(benchmark::State& state) {
428 for (auto _ : state) {
429 }
430 }
431 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
432 ADD_CASES(
433 TC_ConsoleOut,
434 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
435 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
436 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
437 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
438 ADD_CASES(TC_JSONOut,
439 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
440 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
441 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
442 {"\"run_type\": \"aggregate\",$", MR_Next},
443 {"\"aggregate_name\": \"mean\",$", MR_Next},
444 {"\"iterations\": 2,$", MR_Next},
445 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
446 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
447 {"\"run_type\": \"aggregate\",$", MR_Next},
448 {"\"aggregate_name\": \"median\",$", MR_Next},
449 {"\"iterations\": 2,$", MR_Next},
450 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
451 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
452 {"\"run_type\": \"aggregate\",$", MR_Next},
453 {"\"aggregate_name\": \"stddev\",$", MR_Next},
454 {"\"iterations\": 2,$", MR_Next}});
455 ADD_CASES(TC_CSVOut,
456 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
457 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
458 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
459 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
460
461 // Test repeats with custom time unit.
BM_RepeatTimeUnit(benchmark::State & state)462 void BM_RepeatTimeUnit(benchmark::State& state) {
463 for (auto _ : state) {
464 }
465 }
466 BENCHMARK(BM_RepeatTimeUnit)
467 ->Repetitions(3)
468 ->ReportAggregatesOnly()
469 ->Unit(benchmark::kMicrosecond);
470 ADD_CASES(
471 TC_ConsoleOut,
472 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
473 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
474 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
475 "]*3$"},
476 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
477 "]*3$"}});
478 ADD_CASES(TC_JSONOut,
479 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
480 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
481 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
482 {"\"run_type\": \"aggregate\",$", MR_Next},
483 {"\"aggregate_name\": \"mean\",$", MR_Next},
484 {"\"iterations\": 3,$", MR_Next},
485 {"\"time_unit\": \"us\",?$"},
486 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
487 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
488 {"\"run_type\": \"aggregate\",$", MR_Next},
489 {"\"aggregate_name\": \"median\",$", MR_Next},
490 {"\"iterations\": 3,$", MR_Next},
491 {"\"time_unit\": \"us\",?$"},
492 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
493 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
494 {"\"run_type\": \"aggregate\",$", MR_Next},
495 {"\"aggregate_name\": \"stddev\",$", MR_Next},
496 {"\"iterations\": 3,$", MR_Next},
497 {"\"time_unit\": \"us\",?$"}});
498 ADD_CASES(TC_CSVOut,
499 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
500 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
501 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
502 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
503
504 // ========================================================================= //
505 // -------------------- Testing user-provided statistics ------------------- //
506 // ========================================================================= //
507
__anone621c1330102(const std::vector<double>& v) 508 const auto UserStatistics = [](const std::vector<double>& v) {
509 return v.back();
510 };
BM_UserStats(benchmark::State & state)511 void BM_UserStats(benchmark::State& state) {
512 for (auto _ : state) {
513 state.SetIterationTime(150 / 10e8);
514 }
515 }
516 // clang-format off
517 BENCHMARK(BM_UserStats)
518 ->Repetitions(3)
519 ->Iterations(5)
520 ->UseManualTime()
521 ->ComputeStatistics("", UserStatistics);
522 // clang-format on
523
524 // check that user-provided stats is calculated, and is after the default-ones
525 // empty string as name is intentional, it would sort before anything else
526 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
527 "]* 150 ns %time [ ]*5$"},
528 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
529 "]* 150 ns %time [ ]*5$"},
530 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
531 "]* 150 ns %time [ ]*5$"},
532 {"^BM_UserStats/iterations:5/repeats:3/"
533 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
534 {"^BM_UserStats/iterations:5/repeats:3/"
535 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
536 {"^BM_UserStats/iterations:5/repeats:3/"
537 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
538 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
539 "[ ]* 150 ns %time [ ]*3$"}});
540 ADD_CASES(
541 TC_JSONOut,
542 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
543 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
544 MR_Next},
545 {"\"run_type\": \"iteration\",$", MR_Next},
546 {"\"iterations\": 5,$", MR_Next},
547 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
548 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
549 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
550 MR_Next},
551 {"\"run_type\": \"iteration\",$", MR_Next},
552 {"\"iterations\": 5,$", MR_Next},
553 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
554 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
555 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
556 MR_Next},
557 {"\"run_type\": \"iteration\",$", MR_Next},
558 {"\"iterations\": 5,$", MR_Next},
559 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
560 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
561 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
562 MR_Next},
563 {"\"run_type\": \"aggregate\",$", MR_Next},
564 {"\"aggregate_name\": \"mean\",$", MR_Next},
565 {"\"iterations\": 3,$", MR_Next},
566 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
567 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
568 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
569 MR_Next},
570 {"\"run_type\": \"aggregate\",$", MR_Next},
571 {"\"aggregate_name\": \"median\",$", MR_Next},
572 {"\"iterations\": 3,$", MR_Next},
573 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
574 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
575 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
576 MR_Next},
577 {"\"run_type\": \"aggregate\",$", MR_Next},
578 {"\"aggregate_name\": \"stddev\",$", MR_Next},
579 {"\"iterations\": 3,$", MR_Next},
580 {"\"real_time\": %float,$", MR_Next},
581 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
582 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
583 MR_Next},
584 {"\"run_type\": \"aggregate\",$", MR_Next},
585 {"\"aggregate_name\": \"\",$", MR_Next},
586 {"\"iterations\": 3,$", MR_Next},
587 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
588 ADD_CASES(
589 TC_CSVOut,
590 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
591 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
592 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
593 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
594 {"^\"BM_UserStats/iterations:5/repeats:3/"
595 "manual_time_median\",%csv_report$"},
596 {"^\"BM_UserStats/iterations:5/repeats:3/"
597 "manual_time_stddev\",%csv_report$"},
598 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
599
600 // ========================================================================= //
601 // --------------------------- TEST CASES END ------------------------------ //
602 // ========================================================================= //
603
main(int argc,char * argv[])604 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
605