1 
2 #undef NDEBUG
3 #include <cassert>
4 #include <vector>
5 
6 #include "../src/check.h"  // NOTE: check.h is for internal use only!
7 #include "benchmark/benchmark.h"
8 
9 namespace {
10 
11 class TestReporter : public benchmark::ConsoleReporter {
12  public:
ReportContext(const Context & context)13   virtual bool ReportContext(const Context& context) {
14     return ConsoleReporter::ReportContext(context);
15   };
16 
ReportRuns(const std::vector<Run> & report)17   virtual void ReportRuns(const std::vector<Run>& report) {
18     all_runs_.insert(all_runs_.end(), begin(report), end(report));
19     ConsoleReporter::ReportRuns(report);
20   }
21 
TestReporter()22   TestReporter() {}
~TestReporter()23   virtual ~TestReporter() {}
24 
25   mutable std::vector<Run> all_runs_;
26 };
27 
28 struct TestCase {
29   std::string name;
30   bool error_occurred;
31   std::string error_message;
32 
33   typedef benchmark::BenchmarkReporter::Run Run;
34 
CheckRun__anonf46c62d20111::TestCase35   void CheckRun(Run const& run) const {
36     CHECK(name == run.benchmark_name())
37         << "expected " << name << " got " << run.benchmark_name();
38     CHECK(error_occurred == run.error_occurred);
39     CHECK(error_message == run.error_message);
40     if (error_occurred) {
41       // CHECK(run.iterations == 0);
42     } else {
43       CHECK(run.iterations != 0);
44     }
45   }
46 };
47 
48 std::vector<TestCase> ExpectedResults;
49 
AddCases(const char * base_name,std::initializer_list<TestCase> const & v)50 int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
51   for (auto TC : v) {
52     TC.name = base_name + TC.name;
53     ExpectedResults.push_back(std::move(TC));
54   }
55   return 0;
56 }
57 
58 #define CONCAT(x, y) CONCAT2(x, y)
59 #define CONCAT2(x, y) x##y
60 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
61 
62 }  // end namespace
63 
BM_error_no_running(benchmark::State & state)64 void BM_error_no_running(benchmark::State& state) {
65   state.SkipWithError("error message");
66 }
67 BENCHMARK(BM_error_no_running);
68 ADD_CASES("BM_error_no_running", {{"", true, "error message"}});
69 
BM_error_before_running(benchmark::State & state)70 void BM_error_before_running(benchmark::State& state) {
71   state.SkipWithError("error message");
72   while (state.KeepRunning()) {
73     assert(false);
74   }
75 }
76 BENCHMARK(BM_error_before_running);
77 ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
78 
BM_error_before_running_batch(benchmark::State & state)79 void BM_error_before_running_batch(benchmark::State& state) {
80   state.SkipWithError("error message");
81   while (state.KeepRunningBatch(17)) {
82     assert(false);
83   }
84 }
85 BENCHMARK(BM_error_before_running_batch);
86 ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}});
87 
BM_error_before_running_range_for(benchmark::State & state)88 void BM_error_before_running_range_for(benchmark::State& state) {
89   state.SkipWithError("error message");
90   for (auto _ : state) {
91     assert(false);
92   }
93 }
94 BENCHMARK(BM_error_before_running_range_for);
95 ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}});
96 
BM_error_during_running(benchmark::State & state)97 void BM_error_during_running(benchmark::State& state) {
98   int first_iter = true;
99   while (state.KeepRunning()) {
100     if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
101       assert(first_iter);
102       first_iter = false;
103       state.SkipWithError("error message");
104     } else {
105       state.PauseTiming();
106       state.ResumeTiming();
107     }
108   }
109 }
110 BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
111 ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
112                                       {"/1/threads:2", true, "error message"},
113                                       {"/1/threads:4", true, "error message"},
114                                       {"/1/threads:8", true, "error message"},
115                                       {"/2/threads:1", false, ""},
116                                       {"/2/threads:2", false, ""},
117                                       {"/2/threads:4", false, ""},
118                                       {"/2/threads:8", false, ""}});
119 
BM_error_during_running_ranged_for(benchmark::State & state)120 void BM_error_during_running_ranged_for(benchmark::State& state) {
121   assert(state.max_iterations > 3 && "test requires at least a few iterations");
122   int first_iter = true;
123   // NOTE: Users should not write the for loop explicitly.
124   for (auto It = state.begin(), End = state.end(); It != End; ++It) {
125     if (state.range(0) == 1) {
126       assert(first_iter);
127       first_iter = false;
128       state.SkipWithError("error message");
129       // Test the unfortunate but documented behavior that the ranged-for loop
130       // doesn't automatically terminate when SkipWithError is set.
131       assert(++It != End);
132       break;  // Required behavior
133     }
134   }
135 }
136 BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);
137 ADD_CASES("BM_error_during_running_ranged_for",
138           {{"/1/iterations:5", true, "error message"},
139            {"/2/iterations:5", false, ""}});
140 
BM_error_after_running(benchmark::State & state)141 void BM_error_after_running(benchmark::State& state) {
142   for (auto _ : state) {
143     benchmark::DoNotOptimize(state.iterations());
144   }
145   if (state.thread_index <= (state.threads / 2))
146     state.SkipWithError("error message");
147 }
148 BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
149 ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
150                                      {"/threads:2", true, "error message"},
151                                      {"/threads:4", true, "error message"},
152                                      {"/threads:8", true, "error message"}});
153 
BM_error_while_paused(benchmark::State & state)154 void BM_error_while_paused(benchmark::State& state) {
155   bool first_iter = true;
156   while (state.KeepRunning()) {
157     if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
158       assert(first_iter);
159       first_iter = false;
160       state.PauseTiming();
161       state.SkipWithError("error message");
162     } else {
163       state.PauseTiming();
164       state.ResumeTiming();
165     }
166   }
167 }
168 BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
169 ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
170                                     {"/1/threads:2", true, "error message"},
171                                     {"/1/threads:4", true, "error message"},
172                                     {"/1/threads:8", true, "error message"},
173                                     {"/2/threads:1", false, ""},
174                                     {"/2/threads:2", false, ""},
175                                     {"/2/threads:4", false, ""},
176                                     {"/2/threads:8", false, ""}});
177 
main(int argc,char * argv[])178 int main(int argc, char* argv[]) {
179   benchmark::Initialize(&argc, argv);
180 
181   TestReporter test_reporter;
182   benchmark::RunSpecifiedBenchmarks(&test_reporter);
183 
184   typedef benchmark::BenchmarkReporter::Run Run;
185   auto EB = ExpectedResults.begin();
186 
187   for (Run const& run : test_reporter.all_runs_) {
188     assert(EB != ExpectedResults.end());
189     EB->CheckRun(run);
190     ++EB;
191   }
192   assert(EB == ExpectedResults.end());
193 
194   return 0;
195 }
196