1
2 #undef NDEBUG
3 #include <utility>
4
5 #include "benchmark/benchmark.h"
6 #include "output_test.h"
7
8 // ========================================================================= //
9 // ---------------------- Testing Prologue Output -------------------------- //
10 // ========================================================================= //
11
12 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
13 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
14 {"^[-]+$", MR_Next}});
AddContextCases()15 static int AddContextCases() {
16 AddCases(TC_ConsoleErr,
17 {
18 {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
19 {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
20 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
21 });
22 AddCases(TC_JSONOut,
23 {{"^\\{", MR_Default},
24 {"\"context\":", MR_Next},
25 {"\"date\": \"", MR_Next},
26 {"\"host_name\":", MR_Next},
27 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
28 MR_Next},
29 {"\"num_cpus\": %int,$", MR_Next},
30 {"\"mhz_per_cpu\": %float,$", MR_Next},
31 {"\"caches\": \\[$", MR_Default}});
32 auto const& Info = benchmark::CPUInfo::Get();
33 auto const& Caches = Info.caches;
34 if (!Caches.empty()) {
35 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
36 }
37 for (size_t I = 0; I < Caches.size(); ++I) {
38 std::string num_caches_str =
39 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
40 AddCases(TC_ConsoleErr,
41 {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
42 MR_Next}});
43 AddCases(TC_JSONOut, {{"\\{$", MR_Next},
44 {"\"type\": \"", MR_Next},
45 {"\"level\": %int,$", MR_Next},
46 {"\"size\": %int,$", MR_Next},
47 {"\"num_sharing\": %int$", MR_Next},
48 {"}[,]{0,1}$", MR_Next}});
49 }
50 AddCases(TC_JSONOut, {{"],$"}});
51 auto const& LoadAvg = Info.load_avg;
52 if (!LoadAvg.empty()) {
53 AddCases(TC_ConsoleErr,
54 {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
55 }
56 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
57 return 0;
58 }
59 int dummy_register = AddContextCases();
60 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
61
62 // ========================================================================= //
63 // ------------------------ Testing Basic Output --------------------------- //
64 // ========================================================================= //
65
BM_basic(benchmark::State & state)66 void BM_basic(benchmark::State& state) {
67 for (auto _ : state) {
68 }
69 }
70 BENCHMARK(BM_basic);
71
72 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
73 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
74 {"\"run_name\": \"BM_basic\",$", MR_Next},
75 {"\"run_type\": \"iteration\",$", MR_Next},
76 {"\"repetitions\": 0,$", MR_Next},
77 {"\"repetition_index\": 0,$", MR_Next},
78 {"\"threads\": 1,$", MR_Next},
79 {"\"iterations\": %int,$", MR_Next},
80 {"\"real_time\": %float,$", MR_Next},
81 {"\"cpu_time\": %float,$", MR_Next},
82 {"\"time_unit\": \"ns\"$", MR_Next},
83 {"}", MR_Next}});
84 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
85
86 // ========================================================================= //
87 // ------------------------ Testing Bytes per Second Output ---------------- //
88 // ========================================================================= //
89
BM_bytes_per_second(benchmark::State & state)90 void BM_bytes_per_second(benchmark::State& state) {
91 for (auto _ : state) {
92 // This test requires a non-zero CPU time to avoid divide-by-zero
93 benchmark::DoNotOptimize(state.iterations());
94 }
95 state.SetBytesProcessed(1);
96 }
97 BENCHMARK(BM_bytes_per_second);
98
99 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
100 "bytes_per_second=%float[kM]{0,1}/s$"}});
101 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
102 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
103 {"\"run_type\": \"iteration\",$", MR_Next},
104 {"\"repetitions\": 0,$", MR_Next},
105 {"\"repetition_index\": 0,$", MR_Next},
106 {"\"threads\": 1,$", MR_Next},
107 {"\"iterations\": %int,$", MR_Next},
108 {"\"real_time\": %float,$", MR_Next},
109 {"\"cpu_time\": %float,$", MR_Next},
110 {"\"time_unit\": \"ns\",$", MR_Next},
111 {"\"bytes_per_second\": %float$", MR_Next},
112 {"}", MR_Next}});
113 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
114
115 // ========================================================================= //
116 // ------------------------ Testing Items per Second Output ---------------- //
117 // ========================================================================= //
118
BM_items_per_second(benchmark::State & state)119 void BM_items_per_second(benchmark::State& state) {
120 for (auto _ : state) {
121 // This test requires a non-zero CPU time to avoid divide-by-zero
122 benchmark::DoNotOptimize(state.iterations());
123 }
124 state.SetItemsProcessed(1);
125 }
126 BENCHMARK(BM_items_per_second);
127
128 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
129 "items_per_second=%float[kM]{0,1}/s$"}});
130 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
131 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
132 {"\"run_type\": \"iteration\",$", MR_Next},
133 {"\"repetitions\": 0,$", MR_Next},
134 {"\"repetition_index\": 0,$", MR_Next},
135 {"\"threads\": 1,$", MR_Next},
136 {"\"iterations\": %int,$", MR_Next},
137 {"\"real_time\": %float,$", MR_Next},
138 {"\"cpu_time\": %float,$", MR_Next},
139 {"\"time_unit\": \"ns\",$", MR_Next},
140 {"\"items_per_second\": %float$", MR_Next},
141 {"}", MR_Next}});
142 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
143
144 // ========================================================================= //
145 // ------------------------ Testing Label Output --------------------------- //
146 // ========================================================================= //
147
BM_label(benchmark::State & state)148 void BM_label(benchmark::State& state) {
149 for (auto _ : state) {
150 }
151 state.SetLabel("some label");
152 }
153 BENCHMARK(BM_label);
154
155 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
156 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
157 {"\"run_name\": \"BM_label\",$", MR_Next},
158 {"\"run_type\": \"iteration\",$", MR_Next},
159 {"\"repetitions\": 0,$", MR_Next},
160 {"\"repetition_index\": 0,$", MR_Next},
161 {"\"threads\": 1,$", MR_Next},
162 {"\"iterations\": %int,$", MR_Next},
163 {"\"real_time\": %float,$", MR_Next},
164 {"\"cpu_time\": %float,$", MR_Next},
165 {"\"time_unit\": \"ns\",$", MR_Next},
166 {"\"label\": \"some label\"$", MR_Next},
167 {"}", MR_Next}});
168 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
169 "label\"%csv_label_report_end$"}});
170
171 // ========================================================================= //
172 // ------------------------ Testing Time Label Output ---------------------- //
173 // ========================================================================= //
174
BM_time_label_nanosecond(benchmark::State & state)175 void BM_time_label_nanosecond(benchmark::State& state) {
176 for (auto _ : state) {
177 }
178 }
179 BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);
180
181 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
182 ADD_CASES(TC_JSONOut,
183 {{"\"name\": \"BM_time_label_nanosecond\",$"},
184 {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
185 {"\"run_type\": \"iteration\",$", MR_Next},
186 {"\"repetitions\": 0,$", MR_Next},
187 {"\"repetition_index\": 0,$", MR_Next},
188 {"\"threads\": 1,$", MR_Next},
189 {"\"iterations\": %int,$", MR_Next},
190 {"\"real_time\": %float,$", MR_Next},
191 {"\"cpu_time\": %float,$", MR_Next},
192 {"\"time_unit\": \"ns\"$", MR_Next},
193 {"}", MR_Next}});
194 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
195
BM_time_label_microsecond(benchmark::State & state)196 void BM_time_label_microsecond(benchmark::State& state) {
197 for (auto _ : state) {
198 }
199 }
200 BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);
201
202 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
203 ADD_CASES(TC_JSONOut,
204 {{"\"name\": \"BM_time_label_microsecond\",$"},
205 {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
206 {"\"run_type\": \"iteration\",$", MR_Next},
207 {"\"repetitions\": 0,$", MR_Next},
208 {"\"repetition_index\": 0,$", MR_Next},
209 {"\"threads\": 1,$", MR_Next},
210 {"\"iterations\": %int,$", MR_Next},
211 {"\"real_time\": %float,$", MR_Next},
212 {"\"cpu_time\": %float,$", MR_Next},
213 {"\"time_unit\": \"us\"$", MR_Next},
214 {"}", MR_Next}});
215 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
216
BM_time_label_millisecond(benchmark::State & state)217 void BM_time_label_millisecond(benchmark::State& state) {
218 for (auto _ : state) {
219 }
220 }
221 BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);
222
223 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
224 ADD_CASES(TC_JSONOut,
225 {{"\"name\": \"BM_time_label_millisecond\",$"},
226 {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
227 {"\"run_type\": \"iteration\",$", MR_Next},
228 {"\"repetitions\": 0,$", MR_Next},
229 {"\"repetition_index\": 0,$", MR_Next},
230 {"\"threads\": 1,$", MR_Next},
231 {"\"iterations\": %int,$", MR_Next},
232 {"\"real_time\": %float,$", MR_Next},
233 {"\"cpu_time\": %float,$", MR_Next},
234 {"\"time_unit\": \"ms\"$", MR_Next},
235 {"}", MR_Next}});
236 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
237
BM_time_label_second(benchmark::State & state)238 void BM_time_label_second(benchmark::State& state) {
239 for (auto _ : state) {
240 }
241 }
242 BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);
243
244 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
245 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
246 {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
247 {"\"run_type\": \"iteration\",$", MR_Next},
248 {"\"repetitions\": 0,$", MR_Next},
249 {"\"repetition_index\": 0,$", MR_Next},
250 {"\"threads\": 1,$", MR_Next},
251 {"\"iterations\": %int,$", MR_Next},
252 {"\"real_time\": %float,$", MR_Next},
253 {"\"cpu_time\": %float,$", MR_Next},
254 {"\"time_unit\": \"s\"$", MR_Next},
255 {"}", MR_Next}});
256 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
257
258 // ========================================================================= //
259 // ------------------------ Testing Error Output --------------------------- //
260 // ========================================================================= //
261
BM_error(benchmark::State & state)262 void BM_error(benchmark::State& state) {
263 state.SkipWithError("message");
264 for (auto _ : state) {
265 }
266 }
267 BENCHMARK(BM_error);
268 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
269 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
270 {"\"run_name\": \"BM_error\",$", MR_Next},
271 {"\"run_type\": \"iteration\",$", MR_Next},
272 {"\"repetitions\": 0,$", MR_Next},
273 {"\"repetition_index\": 0,$", MR_Next},
274 {"\"threads\": 1,$", MR_Next},
275 {"\"error_occurred\": true,$", MR_Next},
276 {"\"error_message\": \"message\",$", MR_Next}});
277
278 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
279
280 // ========================================================================= //
281 // ------------------------ Testing No Arg Name Output -----------------------
282 // //
283 // ========================================================================= //
284
BM_no_arg_name(benchmark::State & state)285 void BM_no_arg_name(benchmark::State& state) {
286 for (auto _ : state) {
287 }
288 }
289 BENCHMARK(BM_no_arg_name)->Arg(3);
290 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
291 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
292 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
293 {"\"run_type\": \"iteration\",$", MR_Next},
294 {"\"repetitions\": 0,$", MR_Next},
295 {"\"repetition_index\": 0,$", MR_Next},
296 {"\"threads\": 1,$", MR_Next}});
297 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
298
299 // ========================================================================= //
300 // ------------------------ Testing Arg Name Output ----------------------- //
301 // ========================================================================= //
302
BM_arg_name(benchmark::State & state)303 void BM_arg_name(benchmark::State& state) {
304 for (auto _ : state) {
305 }
306 }
307 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
308 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
309 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
310 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
311 {"\"run_type\": \"iteration\",$", MR_Next},
312 {"\"repetitions\": 0,$", MR_Next},
313 {"\"repetition_index\": 0,$", MR_Next},
314 {"\"threads\": 1,$", MR_Next}});
315 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
316
317 // ========================================================================= //
318 // ------------------------ Testing Arg Names Output ----------------------- //
319 // ========================================================================= //
320
BM_arg_names(benchmark::State & state)321 void BM_arg_names(benchmark::State& state) {
322 for (auto _ : state) {
323 }
324 }
325 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
326 ADD_CASES(TC_ConsoleOut,
327 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
328 ADD_CASES(TC_JSONOut,
329 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
330 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
331 {"\"run_type\": \"iteration\",$", MR_Next},
332 {"\"repetitions\": 0,$", MR_Next},
333 {"\"repetition_index\": 0,$", MR_Next},
334 {"\"threads\": 1,$", MR_Next}});
335 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
336
337 // ========================================================================= //
338 // ------------------------ Testing Big Args Output ------------------------ //
339 // ========================================================================= //
340
BM_BigArgs(benchmark::State & state)341 void BM_BigArgs(benchmark::State& state) {
342 for (auto _ : state) {
343 }
344 }
345 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
346 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
347 {"^BM_BigArgs/2147483648 %console_report$"}});
348
349 // ========================================================================= //
350 // ----------------------- Testing Complexity Output ----------------------- //
351 // ========================================================================= //
352
BM_Complexity_O1(benchmark::State & state)353 void BM_Complexity_O1(benchmark::State& state) {
354 for (auto _ : state) {
355 // This test requires a non-zero CPU time to avoid divide-by-zero
356 benchmark::DoNotOptimize(state.iterations());
357 }
358 state.SetComplexityN(state.range(0));
359 }
360 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
361 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
362 {"%RMS", "[ ]*[0-9]+ %"}});
363 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
364 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
365
366 // ========================================================================= //
367 // ----------------------- Testing Aggregate Output ------------------------ //
368 // ========================================================================= //
369
370 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)371 void BM_Repeat(benchmark::State& state) {
372 for (auto _ : state) {
373 }
374 }
375 // need two repetitions min to be able to output any aggregate output
376 BENCHMARK(BM_Repeat)->Repetitions(2);
377 ADD_CASES(TC_ConsoleOut,
378 {{"^BM_Repeat/repeats:2 %console_report$"},
379 {"^BM_Repeat/repeats:2 %console_report$"},
380 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
381 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
382 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
383 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
384 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
385 {"\"run_type\": \"iteration\",$", MR_Next},
386 {"\"repetitions\": 2,$", MR_Next},
387 {"\"repetition_index\": 0,$", MR_Next},
388 {"\"threads\": 1,$", MR_Next},
389 {"\"name\": \"BM_Repeat/repeats:2\",$"},
390 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
391 {"\"run_type\": \"iteration\",$", MR_Next},
392 {"\"repetitions\": 2,$", MR_Next},
393 {"\"repetition_index\": 1,$", MR_Next},
394 {"\"threads\": 1,$", MR_Next},
395 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
396 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
397 {"\"run_type\": \"aggregate\",$", MR_Next},
398 {"\"repetitions\": 2,$", MR_Next},
399 {"\"threads\": 1,$", MR_Next},
400 {"\"aggregate_name\": \"mean\",$", MR_Next},
401 {"\"iterations\": 2,$", MR_Next},
402 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
403 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
404 {"\"run_type\": \"aggregate\",$", MR_Next},
405 {"\"repetitions\": 2,$", MR_Next},
406 {"\"threads\": 1,$", MR_Next},
407 {"\"aggregate_name\": \"median\",$", MR_Next},
408 {"\"iterations\": 2,$", MR_Next},
409 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
410 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
411 {"\"run_type\": \"aggregate\",$", MR_Next},
412 {"\"repetitions\": 2,$", MR_Next},
413 {"\"threads\": 1,$", MR_Next},
414 {"\"aggregate_name\": \"stddev\",$", MR_Next},
415 {"\"iterations\": 2,$", MR_Next}});
416 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
417 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
418 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
419 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
420 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
421 // but for two repetitions, mean and median is the same, so let's repeat..
422 BENCHMARK(BM_Repeat)->Repetitions(3);
423 ADD_CASES(TC_ConsoleOut,
424 {{"^BM_Repeat/repeats:3 %console_report$"},
425 {"^BM_Repeat/repeats:3 %console_report$"},
426 {"^BM_Repeat/repeats:3 %console_report$"},
427 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
428 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
429 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
430 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
431 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
432 {"\"run_type\": \"iteration\",$", MR_Next},
433 {"\"repetitions\": 3,$", MR_Next},
434 {"\"repetition_index\": 0,$", MR_Next},
435 {"\"threads\": 1,$", MR_Next},
436 {"\"name\": \"BM_Repeat/repeats:3\",$"},
437 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
438 {"\"run_type\": \"iteration\",$", MR_Next},
439 {"\"repetitions\": 3,$", MR_Next},
440 {"\"repetition_index\": 1,$", MR_Next},
441 {"\"threads\": 1,$", MR_Next},
442 {"\"name\": \"BM_Repeat/repeats:3\",$"},
443 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
444 {"\"run_type\": \"iteration\",$", MR_Next},
445 {"\"repetitions\": 3,$", MR_Next},
446 {"\"repetition_index\": 2,$", MR_Next},
447 {"\"threads\": 1,$", MR_Next},
448 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
449 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
450 {"\"run_type\": \"aggregate\",$", MR_Next},
451 {"\"repetitions\": 3,$", MR_Next},
452 {"\"threads\": 1,$", MR_Next},
453 {"\"aggregate_name\": \"mean\",$", MR_Next},
454 {"\"iterations\": 3,$", MR_Next},
455 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
456 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
457 {"\"run_type\": \"aggregate\",$", MR_Next},
458 {"\"repetitions\": 3,$", MR_Next},
459 {"\"threads\": 1,$", MR_Next},
460 {"\"aggregate_name\": \"median\",$", MR_Next},
461 {"\"iterations\": 3,$", MR_Next},
462 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
463 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
464 {"\"run_type\": \"aggregate\",$", MR_Next},
465 {"\"repetitions\": 3,$", MR_Next},
466 {"\"threads\": 1,$", MR_Next},
467 {"\"aggregate_name\": \"stddev\",$", MR_Next},
468 {"\"iterations\": 3,$", MR_Next}});
469 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
470 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
471 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
472 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
473 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
474 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
475 // median differs between even/odd number of repetitions, so just to be sure
476 BENCHMARK(BM_Repeat)->Repetitions(4);
477 ADD_CASES(TC_ConsoleOut,
478 {{"^BM_Repeat/repeats:4 %console_report$"},
479 {"^BM_Repeat/repeats:4 %console_report$"},
480 {"^BM_Repeat/repeats:4 %console_report$"},
481 {"^BM_Repeat/repeats:4 %console_report$"},
482 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
483 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
484 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
485 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
486 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
487 {"\"run_type\": \"iteration\",$", MR_Next},
488 {"\"repetitions\": 4,$", MR_Next},
489 {"\"repetition_index\": 0,$", MR_Next},
490 {"\"threads\": 1,$", MR_Next},
491 {"\"name\": \"BM_Repeat/repeats:4\",$"},
492 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
493 {"\"run_type\": \"iteration\",$", MR_Next},
494 {"\"repetitions\": 4,$", MR_Next},
495 {"\"repetition_index\": 1,$", MR_Next},
496 {"\"threads\": 1,$", MR_Next},
497 {"\"name\": \"BM_Repeat/repeats:4\",$"},
498 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
499 {"\"run_type\": \"iteration\",$", MR_Next},
500 {"\"repetitions\": 4,$", MR_Next},
501 {"\"repetition_index\": 2,$", MR_Next},
502 {"\"threads\": 1,$", MR_Next},
503 {"\"name\": \"BM_Repeat/repeats:4\",$"},
504 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
505 {"\"run_type\": \"iteration\",$", MR_Next},
506 {"\"repetitions\": 4,$", MR_Next},
507 {"\"repetition_index\": 3,$", MR_Next},
508 {"\"threads\": 1,$", MR_Next},
509 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
510 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
511 {"\"run_type\": \"aggregate\",$", MR_Next},
512 {"\"repetitions\": 4,$", MR_Next},
513 {"\"threads\": 1,$", MR_Next},
514 {"\"aggregate_name\": \"mean\",$", MR_Next},
515 {"\"iterations\": 4,$", MR_Next},
516 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
517 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
518 {"\"run_type\": \"aggregate\",$", MR_Next},
519 {"\"repetitions\": 4,$", MR_Next},
520 {"\"threads\": 1,$", MR_Next},
521 {"\"aggregate_name\": \"median\",$", MR_Next},
522 {"\"iterations\": 4,$", MR_Next},
523 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
524 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
525 {"\"run_type\": \"aggregate\",$", MR_Next},
526 {"\"repetitions\": 4,$", MR_Next},
527 {"\"threads\": 1,$", MR_Next},
528 {"\"aggregate_name\": \"stddev\",$", MR_Next},
529 {"\"iterations\": 4,$", MR_Next}});
530 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
531 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
532 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
533 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
534 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
535 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
536 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
537
538 // Test that a non-repeated test still prints non-aggregate results even when
539 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)540 void BM_RepeatOnce(benchmark::State& state) {
541 for (auto _ : state) {
542 }
543 }
544 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
545 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
546 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
547 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
548 {"\"run_type\": \"iteration\",$", MR_Next},
549 {"\"repetitions\": 1,$", MR_Next},
550 {"\"repetition_index\": 0,$", MR_Next},
551 {"\"threads\": 1,$", MR_Next}});
552 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
553
554 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)555 void BM_SummaryRepeat(benchmark::State& state) {
556 for (auto _ : state) {
557 }
558 }
559 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
560 ADD_CASES(
561 TC_ConsoleOut,
562 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
563 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
564 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
565 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
566 ADD_CASES(TC_JSONOut,
567 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
568 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
569 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
570 {"\"run_type\": \"aggregate\",$", MR_Next},
571 {"\"repetitions\": 3,$", MR_Next},
572 {"\"threads\": 1,$", MR_Next},
573 {"\"aggregate_name\": \"mean\",$", MR_Next},
574 {"\"iterations\": 3,$", MR_Next},
575 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
576 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
577 {"\"run_type\": \"aggregate\",$", MR_Next},
578 {"\"repetitions\": 3,$", MR_Next},
579 {"\"threads\": 1,$", MR_Next},
580 {"\"aggregate_name\": \"median\",$", MR_Next},
581 {"\"iterations\": 3,$", MR_Next},
582 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
583 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
584 {"\"run_type\": \"aggregate\",$", MR_Next},
585 {"\"repetitions\": 3,$", MR_Next},
586 {"\"threads\": 1,$", MR_Next},
587 {"\"aggregate_name\": \"stddev\",$", MR_Next},
588 {"\"iterations\": 3,$", MR_Next}});
589 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
590 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
591 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
592 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
593
594 // Test that non-aggregate data is not displayed.
595 // NOTE: this test is kinda bad. we are only testing the display output.
596 // But we don't check that the file output still contains everything...
BM_SummaryDisplay(benchmark::State & state)597 void BM_SummaryDisplay(benchmark::State& state) {
598 for (auto _ : state) {
599 }
600 }
601 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
602 ADD_CASES(
603 TC_ConsoleOut,
604 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
605 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
606 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
607 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
608 ADD_CASES(TC_JSONOut,
609 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
610 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
611 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
612 {"\"run_type\": \"aggregate\",$", MR_Next},
613 {"\"repetitions\": 2,$", MR_Next},
614 {"\"threads\": 1,$", MR_Next},
615 {"\"aggregate_name\": \"mean\",$", MR_Next},
616 {"\"iterations\": 2,$", MR_Next},
617 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
618 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
619 {"\"run_type\": \"aggregate\",$", MR_Next},
620 {"\"repetitions\": 2,$", MR_Next},
621 {"\"threads\": 1,$", MR_Next},
622 {"\"aggregate_name\": \"median\",$", MR_Next},
623 {"\"iterations\": 2,$", MR_Next},
624 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
625 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
626 {"\"run_type\": \"aggregate\",$", MR_Next},
627 {"\"repetitions\": 2,$", MR_Next},
628 {"\"threads\": 1,$", MR_Next},
629 {"\"aggregate_name\": \"stddev\",$", MR_Next},
630 {"\"iterations\": 2,$", MR_Next}});
631 ADD_CASES(TC_CSVOut,
632 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
633 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
634 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
635 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
636
637 // Test repeats with custom time unit.
BM_RepeatTimeUnit(benchmark::State & state)638 void BM_RepeatTimeUnit(benchmark::State& state) {
639 for (auto _ : state) {
640 }
641 }
642 BENCHMARK(BM_RepeatTimeUnit)
643 ->Repetitions(3)
644 ->ReportAggregatesOnly()
645 ->Unit(benchmark::kMicrosecond);
646 ADD_CASES(
647 TC_ConsoleOut,
648 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
649 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
650 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
651 "]*3$"},
652 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
653 "]*3$"}});
654 ADD_CASES(TC_JSONOut,
655 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
656 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
657 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
658 {"\"run_type\": \"aggregate\",$", MR_Next},
659 {"\"repetitions\": 3,$", MR_Next},
660 {"\"threads\": 1,$", MR_Next},
661 {"\"aggregate_name\": \"mean\",$", MR_Next},
662 {"\"iterations\": 3,$", MR_Next},
663 {"\"time_unit\": \"us\",?$"},
664 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
665 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
666 {"\"run_type\": \"aggregate\",$", MR_Next},
667 {"\"repetitions\": 3,$", MR_Next},
668 {"\"threads\": 1,$", MR_Next},
669 {"\"aggregate_name\": \"median\",$", MR_Next},
670 {"\"iterations\": 3,$", MR_Next},
671 {"\"time_unit\": \"us\",?$"},
672 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
673 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
674 {"\"run_type\": \"aggregate\",$", MR_Next},
675 {"\"repetitions\": 3,$", MR_Next},
676 {"\"threads\": 1,$", MR_Next},
677 {"\"aggregate_name\": \"stddev\",$", MR_Next},
678 {"\"iterations\": 3,$", MR_Next},
679 {"\"time_unit\": \"us\",?$"}});
680 ADD_CASES(TC_CSVOut,
681 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
682 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
683 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
684 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
685
686 // ========================================================================= //
687 // -------------------- Testing user-provided statistics ------------------- //
688 // ========================================================================= //
689
__anon1e58fb7a0102(const std::vector<double>& v) 690 const auto UserStatistics = [](const std::vector<double>& v) {
691 return v.back();
692 };
BM_UserStats(benchmark::State & state)693 void BM_UserStats(benchmark::State& state) {
694 for (auto _ : state) {
695 state.SetIterationTime(150 / 10e8);
696 }
697 }
698 // clang-format off
699 BENCHMARK(BM_UserStats)
700 ->Repetitions(3)
701 ->Iterations(5)
702 ->UseManualTime()
703 ->ComputeStatistics("", UserStatistics);
704 // clang-format on
705
706 // check that user-provided stats is calculated, and is after the default-ones
707 // empty string as name is intentional, it would sort before anything else
708 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
709 "]* 150 ns %time [ ]*5$"},
710 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
711 "]* 150 ns %time [ ]*5$"},
712 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
713 "]* 150 ns %time [ ]*5$"},
714 {"^BM_UserStats/iterations:5/repeats:3/"
715 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
716 {"^BM_UserStats/iterations:5/repeats:3/"
717 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
718 {"^BM_UserStats/iterations:5/repeats:3/"
719 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
720 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
721 "[ ]* 150 ns %time [ ]*3$"}});
722 ADD_CASES(
723 TC_JSONOut,
724 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
725 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
726 MR_Next},
727 {"\"run_type\": \"iteration\",$", MR_Next},
728 {"\"repetitions\": 3,$", MR_Next},
729 {"\"repetition_index\": 0,$", MR_Next},
730 {"\"threads\": 1,$", MR_Next},
731 {"\"iterations\": 5,$", MR_Next},
732 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
733 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
734 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
735 MR_Next},
736 {"\"run_type\": \"iteration\",$", MR_Next},
737 {"\"repetitions\": 3,$", MR_Next},
738 {"\"repetition_index\": 1,$", MR_Next},
739 {"\"threads\": 1,$", MR_Next},
740 {"\"iterations\": 5,$", MR_Next},
741 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
742 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
743 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
744 MR_Next},
745 {"\"run_type\": \"iteration\",$", MR_Next},
746 {"\"repetitions\": 3,$", MR_Next},
747 {"\"repetition_index\": 2,$", MR_Next},
748 {"\"threads\": 1,$", MR_Next},
749 {"\"iterations\": 5,$", MR_Next},
750 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
751 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
752 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
753 MR_Next},
754 {"\"run_type\": \"aggregate\",$", MR_Next},
755 {"\"repetitions\": 3,$", MR_Next},
756 {"\"threads\": 1,$", MR_Next},
757 {"\"aggregate_name\": \"mean\",$", MR_Next},
758 {"\"iterations\": 3,$", MR_Next},
759 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
760 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
761 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
762 MR_Next},
763 {"\"run_type\": \"aggregate\",$", MR_Next},
764 {"\"repetitions\": 3,$", MR_Next},
765 {"\"threads\": 1,$", MR_Next},
766 {"\"aggregate_name\": \"median\",$", MR_Next},
767 {"\"iterations\": 3,$", MR_Next},
768 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
769 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
770 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
771 MR_Next},
772 {"\"run_type\": \"aggregate\",$", MR_Next},
773 {"\"repetitions\": 3,$", MR_Next},
774 {"\"threads\": 1,$", MR_Next},
775 {"\"aggregate_name\": \"stddev\",$", MR_Next},
776 {"\"iterations\": 3,$", MR_Next},
777 {"\"real_time\": %float,$", MR_Next},
778 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
779 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
780 MR_Next},
781 {"\"run_type\": \"aggregate\",$", MR_Next},
782 {"\"repetitions\": 3,$", MR_Next},
783 {"\"threads\": 1,$", MR_Next},
784 {"\"aggregate_name\": \"\",$", MR_Next},
785 {"\"iterations\": 3,$", MR_Next},
786 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
787 ADD_CASES(
788 TC_CSVOut,
789 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
790 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
791 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
792 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
793 {"^\"BM_UserStats/iterations:5/repeats:3/"
794 "manual_time_median\",%csv_report$"},
795 {"^\"BM_UserStats/iterations:5/repeats:3/"
796 "manual_time_stddev\",%csv_report$"},
797 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
798
799 // ========================================================================= //
800 // ------------------------- Testing StrEscape JSON ------------------------ //
801 // ========================================================================= //
802 #if 0 // enable when csv testing code correctly handles multi-line fields
803 void BM_JSON_Format(benchmark::State& state) {
804 state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
805 for (auto _ : state) {
806 }
807 }
808 BENCHMARK(BM_JSON_Format);
809 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
810 {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
811 {"\"run_type\": \"iteration\",$", MR_Next},
812 {"\"repetitions\": 0,$", MR_Next},
813 {"\"repetition_index\": 0,$", MR_Next},
814 {"\"threads\": 1,$", MR_Next},
815 {"\"error_occurred\": true,$", MR_Next},
816 {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
817 #endif
818 // ========================================================================= //
819 // -------------------------- Testing CsvEscape ---------------------------- //
820 // ========================================================================= //
821
BM_CSV_Format(benchmark::State & state)822 void BM_CSV_Format(benchmark::State& state) {
823 state.SkipWithError("\"freedom\"");
824 for (auto _ : state) {
825 }
826 }
827 BENCHMARK(BM_CSV_Format);
828 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
829
830 // ========================================================================= //
831 // --------------------------- TEST CASES END ------------------------------ //
832 // ========================================================================= //
833
main(int argc,char * argv[])834 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
835