1 
2 #undef NDEBUG
3 #include <utility>
4 
5 #include "benchmark/benchmark.h"
6 #include "output_test.h"
7 
8 // ========================================================================= //
9 // ---------------------- Testing Prologue Output -------------------------- //
10 // ========================================================================= //
11 
12 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
13                           {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
14                           {"^[-]+$", MR_Next}});
AddContextCases()15 static int AddContextCases() {
16   AddCases(TC_ConsoleErr,
17            {
18                {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
19                {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
20                {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
21            });
22   AddCases(TC_JSONOut,
23            {{"^\\{", MR_Default},
24             {"\"context\":", MR_Next},
25             {"\"date\": \"", MR_Next},
26             {"\"host_name\":", MR_Next},
27             {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
28              MR_Next},
29             {"\"num_cpus\": %int,$", MR_Next},
30             {"\"mhz_per_cpu\": %float,$", MR_Next},
31             {"\"caches\": \\[$", MR_Default}});
32   auto const& Info = benchmark::CPUInfo::Get();
33   auto const& Caches = Info.caches;
34   if (!Caches.empty()) {
35     AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
36   }
37   for (size_t I = 0; I < Caches.size(); ++I) {
38     std::string num_caches_str =
39         Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
40     AddCases(TC_ConsoleErr,
41              {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
42                MR_Next}});
43     AddCases(TC_JSONOut, {{"\\{$", MR_Next},
44                           {"\"type\": \"", MR_Next},
45                           {"\"level\": %int,$", MR_Next},
46                           {"\"size\": %int,$", MR_Next},
47                           {"\"num_sharing\": %int$", MR_Next},
48                           {"}[,]{0,1}$", MR_Next}});
49   }
50   AddCases(TC_JSONOut, {{"],$"}});
51   auto const& LoadAvg = Info.load_avg;
52   if (!LoadAvg.empty()) {
53     AddCases(TC_ConsoleErr,
54              {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
55   }
56   AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
57   return 0;
58 }
59 int dummy_register = AddContextCases();
60 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
61 
62 // ========================================================================= //
63 // ------------------------ Testing Basic Output --------------------------- //
64 // ========================================================================= //
65 
BM_basic(benchmark::State & state)66 void BM_basic(benchmark::State& state) {
67   for (auto _ : state) {
68   }
69 }
70 BENCHMARK(BM_basic);
71 
72 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
73 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
74                        {"\"run_name\": \"BM_basic\",$", MR_Next},
75                        {"\"run_type\": \"iteration\",$", MR_Next},
76                        {"\"repetitions\": 0,$", MR_Next},
77                        {"\"repetition_index\": 0,$", MR_Next},
78                        {"\"threads\": 1,$", MR_Next},
79                        {"\"iterations\": %int,$", MR_Next},
80                        {"\"real_time\": %float,$", MR_Next},
81                        {"\"cpu_time\": %float,$", MR_Next},
82                        {"\"time_unit\": \"ns\"$", MR_Next},
83                        {"}", MR_Next}});
84 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
85 
86 // ========================================================================= //
87 // ------------------------ Testing Bytes per Second Output ---------------- //
88 // ========================================================================= //
89 
BM_bytes_per_second(benchmark::State & state)90 void BM_bytes_per_second(benchmark::State& state) {
91   for (auto _ : state) {
92     // This test requires a non-zero CPU time to avoid divide-by-zero
93     benchmark::DoNotOptimize(state.iterations());
94   }
95   state.SetBytesProcessed(1);
96 }
97 BENCHMARK(BM_bytes_per_second);
98 
99 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
100                            "bytes_per_second=%float[kM]{0,1}/s$"}});
101 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
102                        {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
103                        {"\"run_type\": \"iteration\",$", MR_Next},
104                        {"\"repetitions\": 0,$", MR_Next},
105                        {"\"repetition_index\": 0,$", MR_Next},
106                        {"\"threads\": 1,$", MR_Next},
107                        {"\"iterations\": %int,$", MR_Next},
108                        {"\"real_time\": %float,$", MR_Next},
109                        {"\"cpu_time\": %float,$", MR_Next},
110                        {"\"time_unit\": \"ns\",$", MR_Next},
111                        {"\"bytes_per_second\": %float$", MR_Next},
112                        {"}", MR_Next}});
113 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
114 
115 // ========================================================================= //
116 // ------------------------ Testing Items per Second Output ---------------- //
117 // ========================================================================= //
118 
BM_items_per_second(benchmark::State & state)119 void BM_items_per_second(benchmark::State& state) {
120   for (auto _ : state) {
121     // This test requires a non-zero CPU time to avoid divide-by-zero
122     benchmark::DoNotOptimize(state.iterations());
123   }
124   state.SetItemsProcessed(1);
125 }
126 BENCHMARK(BM_items_per_second);
127 
128 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
129                            "items_per_second=%float[kM]{0,1}/s$"}});
130 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
131                        {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
132                        {"\"run_type\": \"iteration\",$", MR_Next},
133                        {"\"repetitions\": 0,$", MR_Next},
134                        {"\"repetition_index\": 0,$", MR_Next},
135                        {"\"threads\": 1,$", MR_Next},
136                        {"\"iterations\": %int,$", MR_Next},
137                        {"\"real_time\": %float,$", MR_Next},
138                        {"\"cpu_time\": %float,$", MR_Next},
139                        {"\"time_unit\": \"ns\",$", MR_Next},
140                        {"\"items_per_second\": %float$", MR_Next},
141                        {"}", MR_Next}});
142 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
143 
144 // ========================================================================= //
145 // ------------------------ Testing Label Output --------------------------- //
146 // ========================================================================= //
147 
BM_label(benchmark::State & state)148 void BM_label(benchmark::State& state) {
149   for (auto _ : state) {
150   }
151   state.SetLabel("some label");
152 }
153 BENCHMARK(BM_label);
154 
155 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
156 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
157                        {"\"run_name\": \"BM_label\",$", MR_Next},
158                        {"\"run_type\": \"iteration\",$", MR_Next},
159                        {"\"repetitions\": 0,$", MR_Next},
160                        {"\"repetition_index\": 0,$", MR_Next},
161                        {"\"threads\": 1,$", MR_Next},
162                        {"\"iterations\": %int,$", MR_Next},
163                        {"\"real_time\": %float,$", MR_Next},
164                        {"\"cpu_time\": %float,$", MR_Next},
165                        {"\"time_unit\": \"ns\",$", MR_Next},
166                        {"\"label\": \"some label\"$", MR_Next},
167                        {"}", MR_Next}});
168 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
169                        "label\"%csv_label_report_end$"}});
170 
171 // ========================================================================= //
172 // ------------------------ Testing Time Label Output ---------------------- //
173 // ========================================================================= //
174 
BM_time_label_nanosecond(benchmark::State & state)175 void BM_time_label_nanosecond(benchmark::State& state) {
176   for (auto _ : state) {
177   }
178 }
179 BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);
180 
181 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
182 ADD_CASES(TC_JSONOut,
183           {{"\"name\": \"BM_time_label_nanosecond\",$"},
184            {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
185            {"\"run_type\": \"iteration\",$", MR_Next},
186            {"\"repetitions\": 0,$", MR_Next},
187            {"\"repetition_index\": 0,$", MR_Next},
188            {"\"threads\": 1,$", MR_Next},
189            {"\"iterations\": %int,$", MR_Next},
190            {"\"real_time\": %float,$", MR_Next},
191            {"\"cpu_time\": %float,$", MR_Next},
192            {"\"time_unit\": \"ns\"$", MR_Next},
193            {"}", MR_Next}});
194 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
195 
BM_time_label_microsecond(benchmark::State & state)196 void BM_time_label_microsecond(benchmark::State& state) {
197   for (auto _ : state) {
198   }
199 }
200 BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);
201 
202 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
203 ADD_CASES(TC_JSONOut,
204           {{"\"name\": \"BM_time_label_microsecond\",$"},
205            {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
206            {"\"run_type\": \"iteration\",$", MR_Next},
207            {"\"repetitions\": 0,$", MR_Next},
208            {"\"repetition_index\": 0,$", MR_Next},
209            {"\"threads\": 1,$", MR_Next},
210            {"\"iterations\": %int,$", MR_Next},
211            {"\"real_time\": %float,$", MR_Next},
212            {"\"cpu_time\": %float,$", MR_Next},
213            {"\"time_unit\": \"us\"$", MR_Next},
214            {"}", MR_Next}});
215 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
216 
BM_time_label_millisecond(benchmark::State & state)217 void BM_time_label_millisecond(benchmark::State& state) {
218   for (auto _ : state) {
219   }
220 }
221 BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);
222 
223 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
224 ADD_CASES(TC_JSONOut,
225           {{"\"name\": \"BM_time_label_millisecond\",$"},
226            {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
227            {"\"run_type\": \"iteration\",$", MR_Next},
228            {"\"repetitions\": 0,$", MR_Next},
229            {"\"repetition_index\": 0,$", MR_Next},
230            {"\"threads\": 1,$", MR_Next},
231            {"\"iterations\": %int,$", MR_Next},
232            {"\"real_time\": %float,$", MR_Next},
233            {"\"cpu_time\": %float,$", MR_Next},
234            {"\"time_unit\": \"ms\"$", MR_Next},
235            {"}", MR_Next}});
236 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
237 
BM_time_label_second(benchmark::State & state)238 void BM_time_label_second(benchmark::State& state) {
239   for (auto _ : state) {
240   }
241 }
242 BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);
243 
244 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
245 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
246                        {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
247                        {"\"run_type\": \"iteration\",$", MR_Next},
248                        {"\"repetitions\": 0,$", MR_Next},
249                        {"\"repetition_index\": 0,$", MR_Next},
250                        {"\"threads\": 1,$", MR_Next},
251                        {"\"iterations\": %int,$", MR_Next},
252                        {"\"real_time\": %float,$", MR_Next},
253                        {"\"cpu_time\": %float,$", MR_Next},
254                        {"\"time_unit\": \"s\"$", MR_Next},
255                        {"}", MR_Next}});
256 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
257 
258 // ========================================================================= //
259 // ------------------------ Testing Error Output --------------------------- //
260 // ========================================================================= //
261 
BM_error(benchmark::State & state)262 void BM_error(benchmark::State& state) {
263   state.SkipWithError("message");
264   for (auto _ : state) {
265   }
266 }
267 BENCHMARK(BM_error);
268 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
269 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
270                        {"\"run_name\": \"BM_error\",$", MR_Next},
271                        {"\"run_type\": \"iteration\",$", MR_Next},
272                        {"\"repetitions\": 0,$", MR_Next},
273                        {"\"repetition_index\": 0,$", MR_Next},
274                        {"\"threads\": 1,$", MR_Next},
275                        {"\"error_occurred\": true,$", MR_Next},
276                        {"\"error_message\": \"message\",$", MR_Next}});
277 
278 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
279 
280 // ========================================================================= //
281 // ------------------------ Testing No Arg Name Output -----------------------
282 // //
283 // ========================================================================= //
284 
BM_no_arg_name(benchmark::State & state)285 void BM_no_arg_name(benchmark::State& state) {
286   for (auto _ : state) {
287   }
288 }
289 BENCHMARK(BM_no_arg_name)->Arg(3);
290 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
291 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
292                        {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
293                        {"\"run_type\": \"iteration\",$", MR_Next},
294                        {"\"repetitions\": 0,$", MR_Next},
295                        {"\"repetition_index\": 0,$", MR_Next},
296                        {"\"threads\": 1,$", MR_Next}});
297 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
298 
299 // ========================================================================= //
300 // ------------------------ Testing Arg Name Output ----------------------- //
301 // ========================================================================= //
302 
BM_arg_name(benchmark::State & state)303 void BM_arg_name(benchmark::State& state) {
304   for (auto _ : state) {
305   }
306 }
307 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
308 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
309 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
310                        {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
311                        {"\"run_type\": \"iteration\",$", MR_Next},
312                        {"\"repetitions\": 0,$", MR_Next},
313                        {"\"repetition_index\": 0,$", MR_Next},
314                        {"\"threads\": 1,$", MR_Next}});
315 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
316 
317 // ========================================================================= //
318 // ------------------------ Testing Arg Names Output ----------------------- //
319 // ========================================================================= //
320 
BM_arg_names(benchmark::State & state)321 void BM_arg_names(benchmark::State& state) {
322   for (auto _ : state) {
323   }
324 }
325 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
326 ADD_CASES(TC_ConsoleOut,
327           {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
328 ADD_CASES(TC_JSONOut,
329           {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
330            {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
331            {"\"run_type\": \"iteration\",$", MR_Next},
332            {"\"repetitions\": 0,$", MR_Next},
333            {"\"repetition_index\": 0,$", MR_Next},
334            {"\"threads\": 1,$", MR_Next}});
335 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
336 
337 // ========================================================================= //
338 // ------------------------ Testing Big Args Output ------------------------ //
339 // ========================================================================= //
340 
BM_BigArgs(benchmark::State & state)341 void BM_BigArgs(benchmark::State& state) {
342   for (auto _ : state) {
343   }
344 }
345 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
346 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
347                           {"^BM_BigArgs/2147483648 %console_report$"}});
348 
349 // ========================================================================= //
350 // ----------------------- Testing Complexity Output ----------------------- //
351 // ========================================================================= //
352 
BM_Complexity_O1(benchmark::State & state)353 void BM_Complexity_O1(benchmark::State& state) {
354   for (auto _ : state) {
355     // This test requires a non-zero CPU time to avoid divide-by-zero
356     benchmark::DoNotOptimize(state.iterations());
357   }
358   state.SetComplexityN(state.range(0));
359 }
360 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
361 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
362                    {"%RMS", "[ ]*[0-9]+ %"}});
363 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
364                           {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
365 
366 // ========================================================================= //
367 // ----------------------- Testing Aggregate Output ------------------------ //
368 // ========================================================================= //
369 
370 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)371 void BM_Repeat(benchmark::State& state) {
372   for (auto _ : state) {
373   }
374 }
375 // need two repetitions min to be able to output any aggregate output
376 BENCHMARK(BM_Repeat)->Repetitions(2);
377 ADD_CASES(TC_ConsoleOut,
378           {{"^BM_Repeat/repeats:2 %console_report$"},
379            {"^BM_Repeat/repeats:2 %console_report$"},
380            {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
381            {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
382            {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
383 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
384                        {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
385                        {"\"run_type\": \"iteration\",$", MR_Next},
386                        {"\"repetitions\": 2,$", MR_Next},
387                        {"\"repetition_index\": 0,$", MR_Next},
388                        {"\"threads\": 1,$", MR_Next},
389                        {"\"name\": \"BM_Repeat/repeats:2\",$"},
390                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
391                        {"\"run_type\": \"iteration\",$", MR_Next},
392                        {"\"repetitions\": 2,$", MR_Next},
393                        {"\"repetition_index\": 1,$", MR_Next},
394                        {"\"threads\": 1,$", MR_Next},
395                        {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
396                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
397                        {"\"run_type\": \"aggregate\",$", MR_Next},
398                        {"\"repetitions\": 2,$", MR_Next},
399                        {"\"threads\": 1,$", MR_Next},
400                        {"\"aggregate_name\": \"mean\",$", MR_Next},
401                        {"\"iterations\": 2,$", MR_Next},
402                        {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
403                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
404                        {"\"run_type\": \"aggregate\",$", MR_Next},
405                        {"\"repetitions\": 2,$", MR_Next},
406                        {"\"threads\": 1,$", MR_Next},
407                        {"\"aggregate_name\": \"median\",$", MR_Next},
408                        {"\"iterations\": 2,$", MR_Next},
409                        {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
410                        {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
411                        {"\"run_type\": \"aggregate\",$", MR_Next},
412                        {"\"repetitions\": 2,$", MR_Next},
413                        {"\"threads\": 1,$", MR_Next},
414                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
415                        {"\"iterations\": 2,$", MR_Next}});
416 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
417                       {"^\"BM_Repeat/repeats:2\",%csv_report$"},
418                       {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
419                       {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
420                       {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
421 // but for two repetitions, mean and median is the same, so let's repeat..
422 BENCHMARK(BM_Repeat)->Repetitions(3);
423 ADD_CASES(TC_ConsoleOut,
424           {{"^BM_Repeat/repeats:3 %console_report$"},
425            {"^BM_Repeat/repeats:3 %console_report$"},
426            {"^BM_Repeat/repeats:3 %console_report$"},
427            {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
428            {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
429            {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
430 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
431                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
432                        {"\"run_type\": \"iteration\",$", MR_Next},
433                        {"\"repetitions\": 3,$", MR_Next},
434                        {"\"repetition_index\": 0,$", MR_Next},
435                        {"\"threads\": 1,$", MR_Next},
436                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
437                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
438                        {"\"run_type\": \"iteration\",$", MR_Next},
439                        {"\"repetitions\": 3,$", MR_Next},
440                        {"\"repetition_index\": 1,$", MR_Next},
441                        {"\"threads\": 1,$", MR_Next},
442                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
443                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
444                        {"\"run_type\": \"iteration\",$", MR_Next},
445                        {"\"repetitions\": 3,$", MR_Next},
446                        {"\"repetition_index\": 2,$", MR_Next},
447                        {"\"threads\": 1,$", MR_Next},
448                        {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
449                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
450                        {"\"run_type\": \"aggregate\",$", MR_Next},
451                        {"\"repetitions\": 3,$", MR_Next},
452                        {"\"threads\": 1,$", MR_Next},
453                        {"\"aggregate_name\": \"mean\",$", MR_Next},
454                        {"\"iterations\": 3,$", MR_Next},
455                        {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
456                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
457                        {"\"run_type\": \"aggregate\",$", MR_Next},
458                        {"\"repetitions\": 3,$", MR_Next},
459                        {"\"threads\": 1,$", MR_Next},
460                        {"\"aggregate_name\": \"median\",$", MR_Next},
461                        {"\"iterations\": 3,$", MR_Next},
462                        {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
463                        {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
464                        {"\"run_type\": \"aggregate\",$", MR_Next},
465                        {"\"repetitions\": 3,$", MR_Next},
466                        {"\"threads\": 1,$", MR_Next},
467                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
468                        {"\"iterations\": 3,$", MR_Next}});
469 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
470                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
471                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
472                       {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
473                       {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
474                       {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
475 // median differs between even/odd number of repetitions, so just to be sure
476 BENCHMARK(BM_Repeat)->Repetitions(4);
477 ADD_CASES(TC_ConsoleOut,
478           {{"^BM_Repeat/repeats:4 %console_report$"},
479            {"^BM_Repeat/repeats:4 %console_report$"},
480            {"^BM_Repeat/repeats:4 %console_report$"},
481            {"^BM_Repeat/repeats:4 %console_report$"},
482            {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
483            {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
484            {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
485 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
486                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
487                        {"\"run_type\": \"iteration\",$", MR_Next},
488                        {"\"repetitions\": 4,$", MR_Next},
489                        {"\"repetition_index\": 0,$", MR_Next},
490                        {"\"threads\": 1,$", MR_Next},
491                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
492                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
493                        {"\"run_type\": \"iteration\",$", MR_Next},
494                        {"\"repetitions\": 4,$", MR_Next},
495                        {"\"repetition_index\": 1,$", MR_Next},
496                        {"\"threads\": 1,$", MR_Next},
497                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
498                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
499                        {"\"run_type\": \"iteration\",$", MR_Next},
500                        {"\"repetitions\": 4,$", MR_Next},
501                        {"\"repetition_index\": 2,$", MR_Next},
502                        {"\"threads\": 1,$", MR_Next},
503                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
504                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
505                        {"\"run_type\": \"iteration\",$", MR_Next},
506                        {"\"repetitions\": 4,$", MR_Next},
507                        {"\"repetition_index\": 3,$", MR_Next},
508                        {"\"threads\": 1,$", MR_Next},
509                        {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
510                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
511                        {"\"run_type\": \"aggregate\",$", MR_Next},
512                        {"\"repetitions\": 4,$", MR_Next},
513                        {"\"threads\": 1,$", MR_Next},
514                        {"\"aggregate_name\": \"mean\",$", MR_Next},
515                        {"\"iterations\": 4,$", MR_Next},
516                        {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
517                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
518                        {"\"run_type\": \"aggregate\",$", MR_Next},
519                        {"\"repetitions\": 4,$", MR_Next},
520                        {"\"threads\": 1,$", MR_Next},
521                        {"\"aggregate_name\": \"median\",$", MR_Next},
522                        {"\"iterations\": 4,$", MR_Next},
523                        {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
524                        {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
525                        {"\"run_type\": \"aggregate\",$", MR_Next},
526                        {"\"repetitions\": 4,$", MR_Next},
527                        {"\"threads\": 1,$", MR_Next},
528                        {"\"aggregate_name\": \"stddev\",$", MR_Next},
529                        {"\"iterations\": 4,$", MR_Next}});
530 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
531                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
532                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
533                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
534                       {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
535                       {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
536                       {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
537 
538 // Test that a non-repeated test still prints non-aggregate results even when
539 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)540 void BM_RepeatOnce(benchmark::State& state) {
541   for (auto _ : state) {
542   }
543 }
544 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
545 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
546 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
547                        {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
548                        {"\"run_type\": \"iteration\",$", MR_Next},
549                        {"\"repetitions\": 1,$", MR_Next},
550                        {"\"repetition_index\": 0,$", MR_Next},
551                        {"\"threads\": 1,$", MR_Next}});
552 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
553 
554 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)555 void BM_SummaryRepeat(benchmark::State& state) {
556   for (auto _ : state) {
557   }
558 }
559 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
560 ADD_CASES(
561     TC_ConsoleOut,
562     {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
563      {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
564      {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
565      {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
566 ADD_CASES(TC_JSONOut,
567           {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
568            {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
569            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
570            {"\"run_type\": \"aggregate\",$", MR_Next},
571            {"\"repetitions\": 3,$", MR_Next},
572            {"\"threads\": 1,$", MR_Next},
573            {"\"aggregate_name\": \"mean\",$", MR_Next},
574            {"\"iterations\": 3,$", MR_Next},
575            {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
576            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
577            {"\"run_type\": \"aggregate\",$", MR_Next},
578            {"\"repetitions\": 3,$", MR_Next},
579            {"\"threads\": 1,$", MR_Next},
580            {"\"aggregate_name\": \"median\",$", MR_Next},
581            {"\"iterations\": 3,$", MR_Next},
582            {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
583            {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
584            {"\"run_type\": \"aggregate\",$", MR_Next},
585            {"\"repetitions\": 3,$", MR_Next},
586            {"\"threads\": 1,$", MR_Next},
587            {"\"aggregate_name\": \"stddev\",$", MR_Next},
588            {"\"iterations\": 3,$", MR_Next}});
589 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
590                       {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
591                       {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
592                       {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
593 
594 // Test that non-aggregate data is not displayed.
595 // NOTE: this test is kinda bad. we are only testing the display output.
596 //       But we don't check that the file output still contains everything...
BM_SummaryDisplay(benchmark::State & state)597 void BM_SummaryDisplay(benchmark::State& state) {
598   for (auto _ : state) {
599   }
600 }
601 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
602 ADD_CASES(
603     TC_ConsoleOut,
604     {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
605      {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
606      {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
607      {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
608 ADD_CASES(TC_JSONOut,
609           {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
610            {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
611            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
612            {"\"run_type\": \"aggregate\",$", MR_Next},
613            {"\"repetitions\": 2,$", MR_Next},
614            {"\"threads\": 1,$", MR_Next},
615            {"\"aggregate_name\": \"mean\",$", MR_Next},
616            {"\"iterations\": 2,$", MR_Next},
617            {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
618            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
619            {"\"run_type\": \"aggregate\",$", MR_Next},
620            {"\"repetitions\": 2,$", MR_Next},
621            {"\"threads\": 1,$", MR_Next},
622            {"\"aggregate_name\": \"median\",$", MR_Next},
623            {"\"iterations\": 2,$", MR_Next},
624            {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
625            {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
626            {"\"run_type\": \"aggregate\",$", MR_Next},
627            {"\"repetitions\": 2,$", MR_Next},
628            {"\"threads\": 1,$", MR_Next},
629            {"\"aggregate_name\": \"stddev\",$", MR_Next},
630            {"\"iterations\": 2,$", MR_Next}});
631 ADD_CASES(TC_CSVOut,
632           {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
633            {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
634            {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
635            {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
636 
637 // Test repeats with custom time unit.
BM_RepeatTimeUnit(benchmark::State & state)638 void BM_RepeatTimeUnit(benchmark::State& state) {
639   for (auto _ : state) {
640   }
641 }
642 BENCHMARK(BM_RepeatTimeUnit)
643     ->Repetitions(3)
644     ->ReportAggregatesOnly()
645     ->Unit(benchmark::kMicrosecond);
646 ADD_CASES(
647     TC_ConsoleOut,
648     {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
649      {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
650      {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
651       "]*3$"},
652      {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
653       "]*3$"}});
654 ADD_CASES(TC_JSONOut,
655           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
656            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
657            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
658            {"\"run_type\": \"aggregate\",$", MR_Next},
659            {"\"repetitions\": 3,$", MR_Next},
660            {"\"threads\": 1,$", MR_Next},
661            {"\"aggregate_name\": \"mean\",$", MR_Next},
662            {"\"iterations\": 3,$", MR_Next},
663            {"\"time_unit\": \"us\",?$"},
664            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
665            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
666            {"\"run_type\": \"aggregate\",$", MR_Next},
667            {"\"repetitions\": 3,$", MR_Next},
668            {"\"threads\": 1,$", MR_Next},
669            {"\"aggregate_name\": \"median\",$", MR_Next},
670            {"\"iterations\": 3,$", MR_Next},
671            {"\"time_unit\": \"us\",?$"},
672            {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
673            {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
674            {"\"run_type\": \"aggregate\",$", MR_Next},
675            {"\"repetitions\": 3,$", MR_Next},
676            {"\"threads\": 1,$", MR_Next},
677            {"\"aggregate_name\": \"stddev\",$", MR_Next},
678            {"\"iterations\": 3,$", MR_Next},
679            {"\"time_unit\": \"us\",?$"}});
680 ADD_CASES(TC_CSVOut,
681           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
682            {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
683            {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
684            {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
685 
686 // ========================================================================= //
687 // -------------------- Testing user-provided statistics ------------------- //
688 // ========================================================================= //
689 
__anon1e58fb7a0102(const std::vector<double>& v) 690 const auto UserStatistics = [](const std::vector<double>& v) {
691   return v.back();
692 };
BM_UserStats(benchmark::State & state)693 void BM_UserStats(benchmark::State& state) {
694   for (auto _ : state) {
695     state.SetIterationTime(150 / 10e8);
696   }
697 }
698 // clang-format off
699 BENCHMARK(BM_UserStats)
700   ->Repetitions(3)
701   ->Iterations(5)
702   ->UseManualTime()
703   ->ComputeStatistics("", UserStatistics);
704 // clang-format on
705 
706 // check that user-provided stats is calculated, and is after the default-ones
707 // empty string as name is intentional, it would sort before anything else
708 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
709                            "]* 150 ns %time [ ]*5$"},
710                           {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
711                            "]* 150 ns %time [ ]*5$"},
712                           {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
713                            "]* 150 ns %time [ ]*5$"},
714                           {"^BM_UserStats/iterations:5/repeats:3/"
715                            "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
716                           {"^BM_UserStats/iterations:5/repeats:3/"
717                            "manual_time_median [ ]* 150 ns %time [ ]*3$"},
718                           {"^BM_UserStats/iterations:5/repeats:3/"
719                            "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
720                           {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
721                            "[ ]* 150 ns %time [ ]*3$"}});
722 ADD_CASES(
723     TC_JSONOut,
724     {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
725      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
726       MR_Next},
727      {"\"run_type\": \"iteration\",$", MR_Next},
728      {"\"repetitions\": 3,$", MR_Next},
729      {"\"repetition_index\": 0,$", MR_Next},
730      {"\"threads\": 1,$", MR_Next},
731      {"\"iterations\": 5,$", MR_Next},
732      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
733      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
734      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
735       MR_Next},
736      {"\"run_type\": \"iteration\",$", MR_Next},
737      {"\"repetitions\": 3,$", MR_Next},
738      {"\"repetition_index\": 1,$", MR_Next},
739      {"\"threads\": 1,$", MR_Next},
740      {"\"iterations\": 5,$", MR_Next},
741      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
742      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
743      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
744       MR_Next},
745      {"\"run_type\": \"iteration\",$", MR_Next},
746      {"\"repetitions\": 3,$", MR_Next},
747      {"\"repetition_index\": 2,$", MR_Next},
748      {"\"threads\": 1,$", MR_Next},
749      {"\"iterations\": 5,$", MR_Next},
750      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
751      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
752      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
753       MR_Next},
754      {"\"run_type\": \"aggregate\",$", MR_Next},
755      {"\"repetitions\": 3,$", MR_Next},
756      {"\"threads\": 1,$", MR_Next},
757      {"\"aggregate_name\": \"mean\",$", MR_Next},
758      {"\"iterations\": 3,$", MR_Next},
759      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
760      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
761      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
762       MR_Next},
763      {"\"run_type\": \"aggregate\",$", MR_Next},
764      {"\"repetitions\": 3,$", MR_Next},
765      {"\"threads\": 1,$", MR_Next},
766      {"\"aggregate_name\": \"median\",$", MR_Next},
767      {"\"iterations\": 3,$", MR_Next},
768      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
769      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
770      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
771       MR_Next},
772      {"\"run_type\": \"aggregate\",$", MR_Next},
773      {"\"repetitions\": 3,$", MR_Next},
774      {"\"threads\": 1,$", MR_Next},
775      {"\"aggregate_name\": \"stddev\",$", MR_Next},
776      {"\"iterations\": 3,$", MR_Next},
777      {"\"real_time\": %float,$", MR_Next},
778      {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
779      {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
780       MR_Next},
781      {"\"run_type\": \"aggregate\",$", MR_Next},
782      {"\"repetitions\": 3,$", MR_Next},
783      {"\"threads\": 1,$", MR_Next},
784      {"\"aggregate_name\": \"\",$", MR_Next},
785      {"\"iterations\": 3,$", MR_Next},
786      {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
787 ADD_CASES(
788     TC_CSVOut,
789     {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
790      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
791      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
792      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
793      {"^\"BM_UserStats/iterations:5/repeats:3/"
794       "manual_time_median\",%csv_report$"},
795      {"^\"BM_UserStats/iterations:5/repeats:3/"
796       "manual_time_stddev\",%csv_report$"},
797      {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
798 
799 // ========================================================================= //
800 // ------------------------- Testing StrEscape JSON ------------------------ //
801 // ========================================================================= //
802 #if 0  // enable when csv testing code correctly handles multi-line fields
803 void BM_JSON_Format(benchmark::State& state) {
804   state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
805   for (auto _ : state) {
806   }
807 }
808 BENCHMARK(BM_JSON_Format);
809 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
810                        {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
811                        {"\"run_type\": \"iteration\",$", MR_Next},
812                        {"\"repetitions\": 0,$", MR_Next},
813                        {"\"repetition_index\": 0,$", MR_Next},
814                        {"\"threads\": 1,$", MR_Next},
815                        {"\"error_occurred\": true,$", MR_Next},
816                        {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
817 #endif
818 // ========================================================================= //
819 // -------------------------- Testing CsvEscape ---------------------------- //
820 // ========================================================================= //
821 
BM_CSV_Format(benchmark::State & state)822 void BM_CSV_Format(benchmark::State& state) {
823   state.SkipWithError("\"freedom\"");
824   for (auto _ : state) {
825   }
826 }
827 BENCHMARK(BM_CSV_Format);
828 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
829 
830 // ========================================================================= //
831 // --------------------------- TEST CASES END ------------------------------ //
832 // ========================================================================= //
833 
main(int argc,char * argv[])834 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
835