1 
2 #undef NDEBUG
3 
4 #include "benchmark/benchmark.h"
5 #include "output_test.h"
6 
7 // @todo: <jpmag> this checks the full output at once; the rule for
8 // CounterSet1 was failing because it was not matching "^[-]+$".
9 // @todo: <jpmag> check that the counters are vertically aligned.
10 ADD_CASES(
11     TC_ConsoleOut,
12     {
13         // keeping these lines long improves readability, so:
14         // clang-format off
15     {"^[-]+$", MR_Next},
16     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
17     {"^[-]+$", MR_Next},
18     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
19     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
20     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
21     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
22     {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
23     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
24     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
25     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
26     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
27     {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
28     {"^[-]+$", MR_Next},
29     {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
30     {"^[-]+$", MR_Next},
31     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
32     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
33     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
34     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
35     {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
36     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
37     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
38     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
39     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
40     {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
41     {"^[-]+$", MR_Next},
42     {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
43     {"^[-]+$", MR_Next},
44     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
45     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
46     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
47     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
48     {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
49         // clang-format on
50     });
51 ADD_CASES(TC_CSVOut, {{"%csv_header,"
52                        "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
53 
54 // ========================================================================= //
55 // ------------------------- Tabular Counters Output ----------------------- //
56 // ========================================================================= //
57 
BM_Counters_Tabular(benchmark::State & state)58 void BM_Counters_Tabular(benchmark::State& state) {
59   for (auto _ : state) {
60   }
61   namespace bm = benchmark;
62   state.counters.insert({
63       {"Foo", {1, bm::Counter::kAvgThreads}},
64       {"Bar", {2, bm::Counter::kAvgThreads}},
65       {"Baz", {4, bm::Counter::kAvgThreads}},
66       {"Bat", {8, bm::Counter::kAvgThreads}},
67       {"Frob", {16, bm::Counter::kAvgThreads}},
68       {"Lob", {32, bm::Counter::kAvgThreads}},
69   });
70 }
71 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
72 ADD_CASES(TC_JSONOut,
73           {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
74            {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next},
75            {"\"run_type\": \"iteration\",$", MR_Next},
76            {"\"repetitions\": 0,$", MR_Next},
77            {"\"repetition_index\": 0,$", MR_Next},
78            {"\"threads\": 1,$", MR_Next},
79            {"\"iterations\": %int,$", MR_Next},
80            {"\"real_time\": %float,$", MR_Next},
81            {"\"cpu_time\": %float,$", MR_Next},
82            {"\"time_unit\": \"ns\",$", MR_Next},
83            {"\"Bar\": %float,$", MR_Next},
84            {"\"Bat\": %float,$", MR_Next},
85            {"\"Baz\": %float,$", MR_Next},
86            {"\"Foo\": %float,$", MR_Next},
87            {"\"Frob\": %float,$", MR_Next},
88            {"\"Lob\": %float$", MR_Next},
89            {"}", MR_Next}});
90 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
91                        "%float,%float,%float,%float,%float,%float$"}});
92 // VS2013 does not allow this function to be passed as a lambda argument
93 // to CHECK_BENCHMARK_RESULTS()
CheckTabular(Results const & e)94 void CheckTabular(Results const& e) {
95   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
96   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
97   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
98   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
99   CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
100   CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
101 }
102 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
103 
104 // ========================================================================= //
105 // -------------------- Tabular+Rate Counters Output ----------------------- //
106 // ========================================================================= //
107 
BM_CounterRates_Tabular(benchmark::State & state)108 void BM_CounterRates_Tabular(benchmark::State& state) {
109   for (auto _ : state) {
110     // This test requires a non-zero CPU time to avoid divide-by-zero
111     benchmark::DoNotOptimize(state.iterations());
112   }
113   namespace bm = benchmark;
114   state.counters.insert({
115       {"Foo", {1, bm::Counter::kAvgThreadsRate}},
116       {"Bar", {2, bm::Counter::kAvgThreadsRate}},
117       {"Baz", {4, bm::Counter::kAvgThreadsRate}},
118       {"Bat", {8, bm::Counter::kAvgThreadsRate}},
119       {"Frob", {16, bm::Counter::kAvgThreadsRate}},
120       {"Lob", {32, bm::Counter::kAvgThreadsRate}},
121   });
122 }
123 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
124 ADD_CASES(TC_JSONOut,
125           {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
126            {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
127             MR_Next},
128            {"\"run_type\": \"iteration\",$", MR_Next},
129            {"\"repetitions\": 0,$", MR_Next},
130            {"\"repetition_index\": 0,$", MR_Next},
131            {"\"threads\": 1,$", MR_Next},
132            {"\"iterations\": %int,$", MR_Next},
133            {"\"real_time\": %float,$", MR_Next},
134            {"\"cpu_time\": %float,$", MR_Next},
135            {"\"time_unit\": \"ns\",$", MR_Next},
136            {"\"Bar\": %float,$", MR_Next},
137            {"\"Bat\": %float,$", MR_Next},
138            {"\"Baz\": %float,$", MR_Next},
139            {"\"Foo\": %float,$", MR_Next},
140            {"\"Frob\": %float,$", MR_Next},
141            {"\"Lob\": %float$", MR_Next},
142            {"}", MR_Next}});
143 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
144                        "%float,%float,%float,%float,%float,%float$"}});
145 // VS2013 does not allow this function to be passed as a lambda argument
146 // to CHECK_BENCHMARK_RESULTS()
CheckTabularRate(Results const & e)147 void CheckTabularRate(Results const& e) {
148   double t = e.DurationCPUTime();
149   CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
150   CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
151   CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
152   CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
153   CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
154   CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
155 }
156 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
157                         &CheckTabularRate);
158 
159 // ========================================================================= //
160 // ------------------------- Tabular Counters Output ----------------------- //
161 // ========================================================================= //
162 
163 // set only some of the counters
BM_CounterSet0_Tabular(benchmark::State & state)164 void BM_CounterSet0_Tabular(benchmark::State& state) {
165   for (auto _ : state) {
166   }
167   namespace bm = benchmark;
168   state.counters.insert({
169       {"Foo", {10, bm::Counter::kAvgThreads}},
170       {"Bar", {20, bm::Counter::kAvgThreads}},
171       {"Baz", {40, bm::Counter::kAvgThreads}},
172   });
173 }
174 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
175 ADD_CASES(TC_JSONOut,
176           {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
177            {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
178            {"\"run_type\": \"iteration\",$", MR_Next},
179            {"\"repetitions\": 0,$", MR_Next},
180            {"\"repetition_index\": 0,$", MR_Next},
181            {"\"threads\": 1,$", MR_Next},
182            {"\"iterations\": %int,$", MR_Next},
183            {"\"real_time\": %float,$", MR_Next},
184            {"\"cpu_time\": %float,$", MR_Next},
185            {"\"time_unit\": \"ns\",$", MR_Next},
186            {"\"Bar\": %float,$", MR_Next},
187            {"\"Baz\": %float,$", MR_Next},
188            {"\"Foo\": %float$", MR_Next},
189            {"}", MR_Next}});
190 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
191                        "%float,,%float,%float,,"}});
192 // VS2013 does not allow this function to be passed as a lambda argument
193 // to CHECK_BENCHMARK_RESULTS()
CheckSet0(Results const & e)194 void CheckSet0(Results const& e) {
195   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
196   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
197   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
198 }
199 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
200 
201 // again.
BM_CounterSet1_Tabular(benchmark::State & state)202 void BM_CounterSet1_Tabular(benchmark::State& state) {
203   for (auto _ : state) {
204   }
205   namespace bm = benchmark;
206   state.counters.insert({
207       {"Foo", {15, bm::Counter::kAvgThreads}},
208       {"Bar", {25, bm::Counter::kAvgThreads}},
209       {"Baz", {45, bm::Counter::kAvgThreads}},
210   });
211 }
212 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
213 ADD_CASES(TC_JSONOut,
214           {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
215            {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
216            {"\"run_type\": \"iteration\",$", MR_Next},
217            {"\"repetitions\": 0,$", MR_Next},
218            {"\"repetition_index\": 0,$", MR_Next},
219            {"\"threads\": 1,$", MR_Next},
220            {"\"iterations\": %int,$", MR_Next},
221            {"\"real_time\": %float,$", MR_Next},
222            {"\"cpu_time\": %float,$", MR_Next},
223            {"\"time_unit\": \"ns\",$", MR_Next},
224            {"\"Bar\": %float,$", MR_Next},
225            {"\"Baz\": %float,$", MR_Next},
226            {"\"Foo\": %float$", MR_Next},
227            {"}", MR_Next}});
228 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
229                        "%float,,%float,%float,,"}});
230 // VS2013 does not allow this function to be passed as a lambda argument
231 // to CHECK_BENCHMARK_RESULTS()
CheckSet1(Results const & e)232 void CheckSet1(Results const& e) {
233   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
234   CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
235   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
236 }
237 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
238 
239 // ========================================================================= //
240 // ------------------------- Tabular Counters Output ----------------------- //
241 // ========================================================================= //
242 
243 // set only some of the counters, different set now.
BM_CounterSet2_Tabular(benchmark::State & state)244 void BM_CounterSet2_Tabular(benchmark::State& state) {
245   for (auto _ : state) {
246   }
247   namespace bm = benchmark;
248   state.counters.insert({
249       {"Foo", {10, bm::Counter::kAvgThreads}},
250       {"Bat", {30, bm::Counter::kAvgThreads}},
251       {"Baz", {40, bm::Counter::kAvgThreads}},
252   });
253 }
254 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
255 ADD_CASES(TC_JSONOut,
256           {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
257            {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
258            {"\"run_type\": \"iteration\",$", MR_Next},
259            {"\"repetitions\": 0,$", MR_Next},
260            {"\"repetition_index\": 0,$", MR_Next},
261            {"\"threads\": 1,$", MR_Next},
262            {"\"iterations\": %int,$", MR_Next},
263            {"\"real_time\": %float,$", MR_Next},
264            {"\"cpu_time\": %float,$", MR_Next},
265            {"\"time_unit\": \"ns\",$", MR_Next},
266            {"\"Bat\": %float,$", MR_Next},
267            {"\"Baz\": %float,$", MR_Next},
268            {"\"Foo\": %float$", MR_Next},
269            {"}", MR_Next}});
270 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
271                        ",%float,%float,%float,,"}});
272 // VS2013 does not allow this function to be passed as a lambda argument
273 // to CHECK_BENCHMARK_RESULTS()
CheckSet2(Results const & e)274 void CheckSet2(Results const& e) {
275   CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
276   CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
277   CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
278 }
279 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
280 
281 // ========================================================================= //
282 // --------------------------- TEST CASES END ------------------------------ //
283 // ========================================================================= //
284 
main(int argc,char * argv[])285 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
286