1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/lite/tools/benchmark/benchmark_model.h"
17
18 #include <iostream>
19 #include <sstream>
20
21 #include "tensorflow/lite/profiling/time.h"
22 #include "tensorflow/lite/tools/benchmark/logging.h"
23
24 namespace {
SleepForSeconds(double sleep_seconds)25 void SleepForSeconds(double sleep_seconds) {
26 if (sleep_seconds <= 0.0) {
27 return;
28 }
29 // If requested, sleep between runs for an arbitrary amount of time.
30 // This can be helpful to determine the effect of mobile processor
31 // scaling and thermal throttling.
32 return tflite::profiling::time::SleepForMicros(
33 static_cast<uint64_t>(sleep_seconds * 1e6));
34 }
35
36 } // namespace
37
38 namespace tflite {
39 namespace benchmark {
40 using tensorflow::Stat;
41
DefaultParams()42 BenchmarkParams BenchmarkModel::DefaultParams() {
43 BenchmarkParams params;
44 params.AddParam("num_runs", BenchmarkParam::Create<int32_t>(50));
45 params.AddParam("min_secs", BenchmarkParam::Create<float>(1.0f));
46 params.AddParam("run_delay", BenchmarkParam::Create<float>(-1.0f));
47 params.AddParam("num_threads", BenchmarkParam::Create<int32_t>(1));
48 params.AddParam("benchmark_name", BenchmarkParam::Create<std::string>(""));
49 params.AddParam("output_prefix", BenchmarkParam::Create<std::string>(""));
50 params.AddParam("warmup_runs", BenchmarkParam::Create<int32_t>(1));
51 params.AddParam("warmup_min_secs", BenchmarkParam::Create<float>(0.5f));
52 return params;
53 }
54
BenchmarkModel()55 BenchmarkModel::BenchmarkModel() : params_(DefaultParams()) {}
56
OnBenchmarkEnd(const BenchmarkResults & results)57 void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults &results) {
58 auto inference_us = results.inference_time_us();
59 auto init_us = results.startup_latency_us();
60 auto warmup_us = results.warmup_time_us();
61 TFLITE_LOG(INFO) << "Average inference timings in us: "
62 << "Warmup: " << warmup_us.avg() << ", "
63 << "Init: " << init_us << ", "
64 << "no stats: " << inference_us.avg();
65 }
66
GetFlags()67 std::vector<Flag> BenchmarkModel::GetFlags() {
68 return {
69 CreateFlag<int32_t>("num_runs", ¶ms_,
70 "minimum number of runs, see also min_secs"),
71 CreateFlag<float>(
72 "min_secs", ¶ms_,
73 "minimum number of seconds to rerun for, potentially making the "
74 "actual number of runs to be greater than num_runs"),
75 CreateFlag<float>("run_delay", ¶ms_, "delay between runs in seconds"),
76 CreateFlag<int32_t>("num_threads", ¶ms_, "number of threads"),
77 CreateFlag<std::string>("benchmark_name", ¶ms_, "benchmark name"),
78 CreateFlag<std::string>("output_prefix", ¶ms_,
79 "benchmark output prefix"),
80 CreateFlag<int32_t>(
81 "warmup_runs", ¶ms_,
82 "minimum number of runs performed on initialization, to "
83 "allow performance characteristics to settle, see also "
84 "warmup_min_secs"),
85 CreateFlag<float>(
86 "warmup_min_secs", ¶ms_,
87 "minimum number of seconds to rerun for, potentially making the "
88 "actual number of warm-up runs to be greater than warmup_runs"),
89 };
90 }
91
LogParams()92 void BenchmarkModel::LogParams() {
93 TFLITE_LOG(INFO) << "Min num runs: [" << params_.Get<int32_t>("num_runs")
94 << "]";
95 TFLITE_LOG(INFO) << "Min runs duration (seconds): ["
96 << params_.Get<float>("min_secs") << "]";
97 TFLITE_LOG(INFO) << "Inter-run delay (seconds): ["
98 << params_.Get<float>("run_delay") << "]";
99 TFLITE_LOG(INFO) << "Num threads: [" << params_.Get<int32_t>("num_threads")
100 << "]";
101 TFLITE_LOG(INFO) << "Benchmark name: ["
102 << params_.Get<std::string>("benchmark_name") << "]";
103 TFLITE_LOG(INFO) << "Output prefix: ["
104 << params_.Get<std::string>("output_prefix") << "]";
105 TFLITE_LOG(INFO) << "Min warmup runs: ["
106 << params_.Get<int32_t>("warmup_runs") << "]";
107 TFLITE_LOG(INFO) << "Min warmup runs duration (seconds): ["
108 << params_.Get<float>("warmup_min_secs") << "]";
109 }
110
PrepareInputData()111 void BenchmarkModel::PrepareInputData() {}
112
ResetInputsAndOutputs()113 void BenchmarkModel::ResetInputsAndOutputs() {}
114
Run(int min_num_times,float min_secs,RunType run_type)115 Stat<int64_t> BenchmarkModel::Run(int min_num_times, float min_secs,
116 RunType run_type) {
117 Stat<int64_t> run_stats;
118 TFLITE_LOG(INFO) << "Running benchmark for at least " << min_num_times
119 << " iterations and at least " << min_secs << " seconds";
120 int64_t min_finish_us =
121 profiling::time::NowMicros() + static_cast<int64_t>(min_secs * 1.e6f);
122 for (int run = 0;
123 run < min_num_times || profiling::time::NowMicros() < min_finish_us;
124 run++) {
125 ResetInputsAndOutputs();
126 listeners_.OnSingleRunStart(run_type);
127 int64_t start_us = profiling::time::NowMicros();
128 RunImpl();
129 int64_t end_us = profiling::time::NowMicros();
130 listeners_.OnSingleRunEnd();
131
132 run_stats.UpdateStat(end_us - start_us);
133 SleepForSeconds(params_.Get<float>("run_delay"));
134 }
135
136 std::stringstream stream;
137 run_stats.OutputToStream(&stream);
138 TFLITE_LOG(INFO) << stream.str() << std::endl;
139
140 return run_stats;
141 }
142
ValidateParams()143 bool BenchmarkModel::ValidateParams() { return true; }
144
Run(int argc,char ** argv)145 void BenchmarkModel::Run(int argc, char **argv) {
146 if (!ParseFlags(argc, argv)) {
147 return;
148 }
149 Run();
150 }
151
Run()152 void BenchmarkModel::Run() {
153 ValidateParams();
154 LogParams();
155
156 int64_t initialization_start_us = profiling::time::NowMicros();
157 Init();
158 int64_t initialization_end_us = profiling::time::NowMicros();
159 int64_t startup_latency_us = initialization_end_us - initialization_start_us;
160 TFLITE_LOG(INFO) << "Initialized session in " << startup_latency_us / 1e3
161 << "ms";
162
163 PrepareInputData();
164 uint64_t input_bytes = ComputeInputBytes();
165 listeners_.OnBenchmarkStart(params_);
166 Stat<int64_t> warmup_time_us =
167 Run(params_.Get<int32_t>("warmup_runs"),
168 params_.Get<float>("warmup_min_secs"), WARMUP);
169 Stat<int64_t> inference_time_us =
170 Run(params_.Get<int32_t>("num_runs"), params_.Get<float>("min_secs"),
171 REGULAR);
172 listeners_.OnBenchmarkEnd(
173 {startup_latency_us, input_bytes, warmup_time_us, inference_time_us});
174 }
175
ParseFlags(int argc,char ** argv)176 bool BenchmarkModel::ParseFlags(int argc, char **argv) {
177 auto flag_list = GetFlags();
178 const bool parse_result =
179 Flags::Parse(&argc, const_cast<const char **>(argv), flag_list);
180 if (!parse_result) {
181 std::string usage = Flags::Usage(argv[0], flag_list);
182 TFLITE_LOG(ERROR) << usage;
183 return false;
184 }
185 return true;
186 }
187
188 } // namespace benchmark
189 } // namespace tflite
190