1
2 #undef NDEBUG
3 #include <cassert>
4 #include <vector>
5
6 #include "../src/check.h" // NOTE: check.h is for internal use only!
7 #include "benchmark/benchmark.h"
8
9 namespace {
10
11 class TestReporter : public benchmark::ConsoleReporter {
12 public:
ReportContext(const Context & context)13 virtual bool ReportContext(const Context& context) {
14 return ConsoleReporter::ReportContext(context);
15 };
16
ReportRuns(const std::vector<Run> & report)17 virtual void ReportRuns(const std::vector<Run>& report) {
18 all_runs_.insert(all_runs_.end(), begin(report), end(report));
19 ConsoleReporter::ReportRuns(report);
20 }
21
TestReporter()22 TestReporter() {}
~TestReporter()23 virtual ~TestReporter() {}
24
25 mutable std::vector<Run> all_runs_;
26 };
27
28 struct TestCase {
29 std::string name;
30 bool error_occurred;
31 std::string error_message;
32
33 typedef benchmark::BenchmarkReporter::Run Run;
34
CheckRun__anone3797ca00111::TestCase35 void CheckRun(Run const& run) const {
36 CHECK(name == run.benchmark_name) << "expected " << name << " got "
37 << run.benchmark_name;
38 CHECK(error_occurred == run.error_occurred);
39 CHECK(error_message == run.error_message);
40 if (error_occurred) {
41 // CHECK(run.iterations == 0);
42 } else {
43 CHECK(run.iterations != 0);
44 }
45 }
46 };
47
48 std::vector<TestCase> ExpectedResults;
49
AddCases(const char * base_name,std::initializer_list<TestCase> const & v)50 int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
51 for (auto TC : v) {
52 TC.name = base_name + TC.name;
53 ExpectedResults.push_back(std::move(TC));
54 }
55 return 0;
56 }
57
58 #define CONCAT(x, y) CONCAT2(x, y)
59 #define CONCAT2(x, y) x##y
60 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
61
62 } // end namespace
63
BM_error_before_running(benchmark::State & state)64 void BM_error_before_running(benchmark::State& state) {
65 state.SkipWithError("error message");
66 while (state.KeepRunning()) {
67 assert(false);
68 }
69 }
70 BENCHMARK(BM_error_before_running);
71 ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
72
73
BM_error_before_running_batch(benchmark::State & state)74 void BM_error_before_running_batch(benchmark::State& state) {
75 state.SkipWithError("error message");
76 while (state.KeepRunningBatch(17)) {
77 assert(false);
78 }
79 }
80 BENCHMARK(BM_error_before_running_batch);
81 ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}});
82
BM_error_before_running_range_for(benchmark::State & state)83 void BM_error_before_running_range_for(benchmark::State& state) {
84 state.SkipWithError("error message");
85 for (auto _ : state) {
86 assert(false);
87 }
88 }
89 BENCHMARK(BM_error_before_running_range_for);
90 ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}});
91
BM_error_during_running(benchmark::State & state)92 void BM_error_during_running(benchmark::State& state) {
93 int first_iter = true;
94 while (state.KeepRunning()) {
95 if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
96 assert(first_iter);
97 first_iter = false;
98 state.SkipWithError("error message");
99 } else {
100 state.PauseTiming();
101 state.ResumeTiming();
102 }
103 }
104 }
105 BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
106 ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
107 {"/1/threads:2", true, "error message"},
108 {"/1/threads:4", true, "error message"},
109 {"/1/threads:8", true, "error message"},
110 {"/2/threads:1", false, ""},
111 {"/2/threads:2", false, ""},
112 {"/2/threads:4", false, ""},
113 {"/2/threads:8", false, ""}});
114
BM_error_during_running_ranged_for(benchmark::State & state)115 void BM_error_during_running_ranged_for(benchmark::State& state) {
116 assert(state.max_iterations > 3 && "test requires at least a few iterations");
117 int first_iter = true;
118 // NOTE: Users should not write the for loop explicitly.
119 for (auto It = state.begin(), End = state.end(); It != End; ++It) {
120 if (state.range(0) == 1) {
121 assert(first_iter);
122 first_iter = false;
123 state.SkipWithError("error message");
124 // Test the unfortunate but documented behavior that the ranged-for loop
125 // doesn't automatically terminate when SkipWithError is set.
126 assert(++It != End);
127 break; // Required behavior
128 }
129 }
130 }
131 BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);
132 ADD_CASES("BM_error_during_running_ranged_for",
133 {{"/1/iterations:5", true, "error message"},
134 {"/2/iterations:5", false, ""}});
135
136
137
BM_error_after_running(benchmark::State & state)138 void BM_error_after_running(benchmark::State& state) {
139 for (auto _ : state) {
140 benchmark::DoNotOptimize(state.iterations());
141 }
142 if (state.thread_index <= (state.threads / 2))
143 state.SkipWithError("error message");
144 }
145 BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
146 ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
147 {"/threads:2", true, "error message"},
148 {"/threads:4", true, "error message"},
149 {"/threads:8", true, "error message"}});
150
BM_error_while_paused(benchmark::State & state)151 void BM_error_while_paused(benchmark::State& state) {
152 bool first_iter = true;
153 while (state.KeepRunning()) {
154 if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
155 assert(first_iter);
156 first_iter = false;
157 state.PauseTiming();
158 state.SkipWithError("error message");
159 } else {
160 state.PauseTiming();
161 state.ResumeTiming();
162 }
163 }
164 }
165 BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
166 ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
167 {"/1/threads:2", true, "error message"},
168 {"/1/threads:4", true, "error message"},
169 {"/1/threads:8", true, "error message"},
170 {"/2/threads:1", false, ""},
171 {"/2/threads:2", false, ""},
172 {"/2/threads:4", false, ""},
173 {"/2/threads:8", false, ""}});
174
main(int argc,char * argv[])175 int main(int argc, char* argv[]) {
176 benchmark::Initialize(&argc, argv);
177
178 TestReporter test_reporter;
179 benchmark::RunSpecifiedBenchmarks(&test_reporter);
180
181 typedef benchmark::BenchmarkReporter::Run Run;
182 auto EB = ExpectedResults.begin();
183
184 for (Run const& run : test_reporter.all_runs_) {
185 assert(EB != ExpectedResults.end());
186 EB->CheckRun(run);
187 ++EB;
188 }
189 assert(EB == ExpectedResults.end());
190
191 return 0;
192 }
193