1 
2 #undef NDEBUG
3 
4 #include "benchmark/benchmark.h"
5 #include "output_test.h"
6 
7 // ========================================================================= //
8 // ---------------------- Testing Prologue Output -------------------------- //
9 // ========================================================================= //
10 
11 // clang-format off
12 
13 ADD_CASES(TC_ConsoleOut,
14           {{"^[-]+$", MR_Next},
15            {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next},
16            {"^[-]+$", MR_Next}});
17 ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}});
18 
19 // clang-format on
20 
21 // ========================================================================= //
22 // ------------------------- Simple Counters Output ------------------------ //
23 // ========================================================================= //
24 
BM_Counters_Simple(benchmark::State & state)25 void BM_Counters_Simple(benchmark::State& state) {
26   for (auto _ : state) {
27   }
28   state.counters["foo"] = 1;
29   state.counters["bar"] = 2 * (double)state.iterations();
30 }
31 BENCHMARK(BM_Counters_Simple);
32 ADD_CASES(TC_ConsoleOut,
33           {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
34 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
35                        {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next},
36                        {"\"run_type\": \"iteration\",$", MR_Next},
37                        {"\"iterations\": %int,$", MR_Next},
38                        {"\"real_time\": %float,$", MR_Next},
39                        {"\"cpu_time\": %float,$", MR_Next},
40                        {"\"time_unit\": \"ns\",$", MR_Next},
41                        {"\"bar\": %float,$", MR_Next},
42                        {"\"foo\": %float$", MR_Next},
43                        {"}", MR_Next}});
44 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}});
45 // VS2013 does not allow this function to be passed as a lambda argument
46 // to CHECK_BENCHMARK_RESULTS()
CheckSimple(Results const & e)47 void CheckSimple(Results const& e) {
48   double its = e.NumIterations();
49   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
50   // check that the value of bar is within 0.1% of the expected value
51   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001);
52 }
53 CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple);
54 
55 // ========================================================================= //
56 // --------------------- Counters+Items+Bytes/s Output --------------------- //
57 // ========================================================================= //
58 
59 namespace {
60 int num_calls1 = 0;
61 }
BM_Counters_WithBytesAndItemsPSec(benchmark::State & state)62 void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
63   for (auto _ : state) {
64   }
65   state.counters["foo"] = 1;
66   state.counters["bar"] = ++num_calls1;
67   state.SetBytesProcessed(364);
68   state.SetItemsProcessed(150);
69 }
70 BENCHMARK(BM_Counters_WithBytesAndItemsPSec);
71 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
72                            "bar=%hrfloat bytes_per_second=%hrfloat/s "
73                            "foo=%hrfloat items_per_second=%hrfloat/s$"}});
74 ADD_CASES(TC_JSONOut,
75           {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
76            {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next},
77            {"\"run_type\": \"iteration\",$", MR_Next},
78            {"\"iterations\": %int,$", MR_Next},
79            {"\"real_time\": %float,$", MR_Next},
80            {"\"cpu_time\": %float,$", MR_Next},
81            {"\"time_unit\": \"ns\",$", MR_Next},
82            {"\"bar\": %float,$", MR_Next},
83            {"\"bytes_per_second\": %float,$", MR_Next},
84            {"\"foo\": %float,$", MR_Next},
85            {"\"items_per_second\": %float$", MR_Next},
86            {"}", MR_Next}});
87 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
88                        "%csv_bytes_items_report,%float,%float$"}});
89 // VS2013 does not allow this function to be passed as a lambda argument
90 // to CHECK_BENCHMARK_RESULTS()
CheckBytesAndItemsPSec(Results const & e)91 void CheckBytesAndItemsPSec(Results const& e) {
92   double t = e.DurationCPUTime();  // this (and not real time) is the time used
93   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
94   CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
95   // check that the values are within 0.1% of the expected values
96   CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364. / t, 0.001);
97   CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150. / t, 0.001);
98 }
99 CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
100                         &CheckBytesAndItemsPSec);
101 
102 // ========================================================================= //
103 // ------------------------- Rate Counters Output -------------------------- //
104 // ========================================================================= //
105 
BM_Counters_Rate(benchmark::State & state)106 void BM_Counters_Rate(benchmark::State& state) {
107   for (auto _ : state) {
108   }
109   namespace bm = benchmark;
110   state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate};
111   state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
112 }
113 BENCHMARK(BM_Counters_Rate);
114 ADD_CASES(
115     TC_ConsoleOut,
116     {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
117 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
118                        {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next},
119                        {"\"run_type\": \"iteration\",$", MR_Next},
120                        {"\"iterations\": %int,$", MR_Next},
121                        {"\"real_time\": %float,$", MR_Next},
122                        {"\"cpu_time\": %float,$", MR_Next},
123                        {"\"time_unit\": \"ns\",$", MR_Next},
124                        {"\"bar\": %float,$", MR_Next},
125                        {"\"foo\": %float$", MR_Next},
126                        {"}", MR_Next}});
127 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}});
128 // VS2013 does not allow this function to be passed as a lambda argument
129 // to CHECK_BENCHMARK_RESULTS()
CheckRate(Results const & e)130 void CheckRate(Results const& e) {
131   double t = e.DurationCPUTime();  // this (and not real time) is the time used
132   // check that the values are within 0.1% of the expected values
133   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / t, 0.001);
134   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001);
135 }
136 CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate);
137 
138 // ========================================================================= //
139 // ------------------------- Thread Counters Output ------------------------ //
140 // ========================================================================= //
141 
BM_Counters_Threads(benchmark::State & state)142 void BM_Counters_Threads(benchmark::State& state) {
143   for (auto _ : state) {
144   }
145   state.counters["foo"] = 1;
146   state.counters["bar"] = 2;
147 }
148 BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
149 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report "
150                            "bar=%hrfloat foo=%hrfloat$"}});
151 ADD_CASES(TC_JSONOut,
152           {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
153            {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next},
154            {"\"run_type\": \"iteration\",$", MR_Next},
155            {"\"iterations\": %int,$", MR_Next},
156            {"\"real_time\": %float,$", MR_Next},
157            {"\"cpu_time\": %float,$", MR_Next},
158            {"\"time_unit\": \"ns\",$", MR_Next},
159            {"\"bar\": %float,$", MR_Next},
160            {"\"foo\": %float$", MR_Next},
161            {"}", MR_Next}});
162 ADD_CASES(
163     TC_CSVOut,
164     {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}});
165 // VS2013 does not allow this function to be passed as a lambda argument
166 // to CHECK_BENCHMARK_RESULTS()
CheckThreads(Results const & e)167 void CheckThreads(Results const& e) {
168   CHECK_COUNTER_VALUE(e, int, "foo", EQ, e.NumThreads());
169   CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads());
170 }
171 CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads);
172 
173 // ========================================================================= //
174 // ---------------------- ThreadAvg Counters Output ------------------------ //
175 // ========================================================================= //
176 
BM_Counters_AvgThreads(benchmark::State & state)177 void BM_Counters_AvgThreads(benchmark::State& state) {
178   for (auto _ : state) {
179   }
180   namespace bm = benchmark;
181   state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreads};
182   state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads};
183 }
184 BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
185 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int "
186                            "%console_report bar=%hrfloat foo=%hrfloat$"}});
187 ADD_CASES(TC_JSONOut,
188           {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
189            {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next},
190            {"\"run_type\": \"iteration\",$", MR_Next},
191            {"\"iterations\": %int,$", MR_Next},
192            {"\"real_time\": %float,$", MR_Next},
193            {"\"cpu_time\": %float,$", MR_Next},
194            {"\"time_unit\": \"ns\",$", MR_Next},
195            {"\"bar\": %float,$", MR_Next},
196            {"\"foo\": %float$", MR_Next},
197            {"}", MR_Next}});
198 ADD_CASES(
199     TC_CSVOut,
200     {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}});
201 // VS2013 does not allow this function to be passed as a lambda argument
202 // to CHECK_BENCHMARK_RESULTS()
CheckAvgThreads(Results const & e)203 void CheckAvgThreads(Results const& e) {
204   CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
205   CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2);
206 }
207 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int",
208                         &CheckAvgThreads);
209 
210 // ========================================================================= //
211 // ---------------------- ThreadAvg Counters Output ------------------------ //
212 // ========================================================================= //
213 
BM_Counters_AvgThreadsRate(benchmark::State & state)214 void BM_Counters_AvgThreadsRate(benchmark::State& state) {
215   for (auto _ : state) {
216   }
217   namespace bm = benchmark;
218   state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate};
219   state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};
220 }
221 BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);
222 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int "
223                            "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
224 ADD_CASES(TC_JSONOut,
225           {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
226            {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$",
227             MR_Next},
228            {"\"run_type\": \"iteration\",$", MR_Next},
229            {"\"iterations\": %int,$", MR_Next},
230            {"\"real_time\": %float,$", MR_Next},
231            {"\"cpu_time\": %float,$", MR_Next},
232            {"\"time_unit\": \"ns\",$", MR_Next},
233            {"\"bar\": %float,$", MR_Next},
234            {"\"foo\": %float$", MR_Next},
235            {"}", MR_Next}});
236 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/"
237                        "threads:%int\",%csv_report,%float,%float$"}});
238 // VS2013 does not allow this function to be passed as a lambda argument
239 // to CHECK_BENCHMARK_RESULTS()
CheckAvgThreadsRate(Results const & e)240 void CheckAvgThreadsRate(Results const& e) {
241   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / e.DurationCPUTime(), 0.001);
242   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / e.DurationCPUTime(), 0.001);
243 }
244 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int",
245                         &CheckAvgThreadsRate);
246 
247 // ========================================================================= //
248 // ------------------- IterationInvariant Counters Output ------------------ //
249 // ========================================================================= //
250 
BM_Counters_IterationInvariant(benchmark::State & state)251 void BM_Counters_IterationInvariant(benchmark::State& state) {
252   for (auto _ : state) {
253   }
254   namespace bm = benchmark;
255   state.counters["foo"] = bm::Counter{1, bm::Counter::kIsIterationInvariant};
256   state.counters["bar"] = bm::Counter{2, bm::Counter::kIsIterationInvariant};
257 }
258 BENCHMARK(BM_Counters_IterationInvariant);
259 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report "
260                            "bar=%hrfloat foo=%hrfloat$"}});
261 ADD_CASES(TC_JSONOut,
262           {{"\"name\": \"BM_Counters_IterationInvariant\",$"},
263            {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next},
264            {"\"run_type\": \"iteration\",$", MR_Next},
265            {"\"iterations\": %int,$", MR_Next},
266            {"\"real_time\": %float,$", MR_Next},
267            {"\"cpu_time\": %float,$", MR_Next},
268            {"\"time_unit\": \"ns\",$", MR_Next},
269            {"\"bar\": %float,$", MR_Next},
270            {"\"foo\": %float$", MR_Next},
271            {"}", MR_Next}});
272 ADD_CASES(TC_CSVOut,
273           {{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}});
274 // VS2013 does not allow this function to be passed as a lambda argument
275 // to CHECK_BENCHMARK_RESULTS()
CheckIterationInvariant(Results const & e)276 void CheckIterationInvariant(Results const& e) {
277   double its = e.NumIterations();
278   // check that the values are within 0.1% of the expected value
279   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its, 0.001);
280   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001);
281 }
282 CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant",
283                         &CheckIterationInvariant);
284 
285 // ========================================================================= //
286 // ----------------- IterationInvariantRate Counters Output ---------------- //
287 // ========================================================================= //
288 
BM_Counters_kIsIterationInvariantRate(benchmark::State & state)289 void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) {
290   for (auto _ : state) {
291   }
292   namespace bm = benchmark;
293   state.counters["foo"] =
294       bm::Counter{1, bm::Counter::kIsIterationInvariantRate};
295   state.counters["bar"] =
296       bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant};
297 }
298 BENCHMARK(BM_Counters_kIsIterationInvariantRate);
299 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate "
300                            "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
301 ADD_CASES(TC_JSONOut,
302           {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"},
303            {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$",
304             MR_Next},
305            {"\"run_type\": \"iteration\",$", MR_Next},
306            {"\"iterations\": %int,$", MR_Next},
307            {"\"real_time\": %float,$", MR_Next},
308            {"\"cpu_time\": %float,$", MR_Next},
309            {"\"time_unit\": \"ns\",$", MR_Next},
310            {"\"bar\": %float,$", MR_Next},
311            {"\"foo\": %float$", MR_Next},
312            {"}", MR_Next}});
313 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kIsIterationInvariantRate\",%csv_report,"
314                        "%float,%float$"}});
315 // VS2013 does not allow this function to be passed as a lambda argument
316 // to CHECK_BENCHMARK_RESULTS()
CheckIsIterationInvariantRate(Results const & e)317 void CheckIsIterationInvariantRate(Results const& e) {
318   double its = e.NumIterations();
319   double t = e.DurationCPUTime();  // this (and not real time) is the time used
320   // check that the values are within 0.1% of the expected values
321   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its * 1. / t, 0.001);
322   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, its * 2. / t, 0.001);
323 }
324 CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate",
325                         &CheckIsIterationInvariantRate);
326 
327 // ========================================================================= //
328 // ------------------- AvgIterations Counters Output ------------------ //
329 // ========================================================================= //
330 
BM_Counters_AvgIterations(benchmark::State & state)331 void BM_Counters_AvgIterations(benchmark::State& state) {
332   for (auto _ : state) {
333   }
334   namespace bm = benchmark;
335   state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterations};
336   state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgIterations};
337 }
338 BENCHMARK(BM_Counters_AvgIterations);
339 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report "
340                            "bar=%hrfloat foo=%hrfloat$"}});
341 ADD_CASES(TC_JSONOut,
342           {{"\"name\": \"BM_Counters_AvgIterations\",$"},
343            {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next},
344            {"\"run_type\": \"iteration\",$", MR_Next},
345            {"\"iterations\": %int,$", MR_Next},
346            {"\"real_time\": %float,$", MR_Next},
347            {"\"cpu_time\": %float,$", MR_Next},
348            {"\"time_unit\": \"ns\",$", MR_Next},
349            {"\"bar\": %float,$", MR_Next},
350            {"\"foo\": %float$", MR_Next},
351            {"}", MR_Next}});
352 ADD_CASES(TC_CSVOut,
353           {{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}});
354 // VS2013 does not allow this function to be passed as a lambda argument
355 // to CHECK_BENCHMARK_RESULTS()
CheckAvgIterations(Results const & e)356 void CheckAvgIterations(Results const& e) {
357   double its = e.NumIterations();
358   // check that the values are within 0.1% of the expected value
359   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its, 0.001);
360   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its, 0.001);
361 }
362 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations);
363 
364 // ========================================================================= //
365 // ----------------- AvgIterationsRate Counters Output ---------------- //
366 // ========================================================================= //
367 
BM_Counters_kAvgIterationsRate(benchmark::State & state)368 void BM_Counters_kAvgIterationsRate(benchmark::State& state) {
369   for (auto _ : state) {
370   }
371   namespace bm = benchmark;
372   state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate};
373   state.counters["bar"] =
374       bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations};
375 }
376 BENCHMARK(BM_Counters_kAvgIterationsRate);
377 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate "
378                            "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
379 ADD_CASES(TC_JSONOut,
380           {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"},
381            {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next},
382            {"\"run_type\": \"iteration\",$", MR_Next},
383            {"\"iterations\": %int,$", MR_Next},
384            {"\"real_time\": %float,$", MR_Next},
385            {"\"cpu_time\": %float,$", MR_Next},
386            {"\"time_unit\": \"ns\",$", MR_Next},
387            {"\"bar\": %float,$", MR_Next},
388            {"\"foo\": %float$", MR_Next},
389            {"}", MR_Next}});
390 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report,"
391                        "%float,%float$"}});
392 // VS2013 does not allow this function to be passed as a lambda argument
393 // to CHECK_BENCHMARK_RESULTS()
CheckAvgIterationsRate(Results const & e)394 void CheckAvgIterationsRate(Results const& e) {
395   double its = e.NumIterations();
396   double t = e.DurationCPUTime();  // this (and not real time) is the time used
397   // check that the values are within 0.1% of the expected values
398   CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its / t, 0.001);
399   CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its / t, 0.001);
400 }
401 CHECK_BENCHMARK_RESULTS("BM_Counters_kAvgIterationsRate",
402                         &CheckAvgIterationsRate);
403 
404 // ========================================================================= //
405 // --------------------------- TEST CASES END ------------------------------ //
406 // ========================================================================= //
407 
main(int argc,char * argv[])408 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
409