1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/util/stats_calculator.h"
17
18 #include <iomanip>
19 #include <map>
20 #include <queue>
21 #include <sstream>
22 #include <string>
23
24 namespace tensorflow {
25
StatsCalculator(const StatSummarizerOptions & options)26 StatsCalculator::StatsCalculator(const StatSummarizerOptions& options)
27 : options_(options) {}
28
GetShortSummary() const29 std::string StatsCalculator::GetShortSummary() const {
30 std::stringstream stream;
31 stream << "Timings (microseconds): ";
32 run_total_us_.OutputToStream(&stream);
33 stream << std::endl;
34
35 stream << "Memory (bytes): ";
36 memory_.OutputToStream(&stream);
37 stream << std::endl;
38
39 stream << details_.size() << " nodes observed" << std::endl;
40 return stream.str();
41 }
42
InitField(std::ostream & stream,int width)43 std::ostream& InitField(std::ostream& stream, int width) {
44 stream << "\t" << std::right << std::setw(width) << std::fixed
45 << std::setprecision(3);
46 return stream;
47 }
48
HeaderString(const std::string & title) const49 std::string StatsCalculator::HeaderString(const std::string& title) const {
50 std::stringstream stream;
51
52 stream << "============================== " << title
53 << " ==============================" << std::endl;
54 if (options_.format_as_csv) {
55 stream << "node type, start, first, avg_ms, %, cdf%, mem KB, times called, "
56 "name";
57 } else {
58 InitField(stream, 24) << "[node type]";
59 InitField(stream, 17) << "[start]";
60 InitField(stream, 9) << "[first]";
61 InitField(stream, 9) << "[avg ms]";
62 InitField(stream, 8) << "[%]";
63 InitField(stream, 8) << "[cdf%]";
64 InitField(stream, 10) << "[mem KB]";
65 InitField(stream, 9) << "[times called]";
66 stream << "\t"
67 << "[Name]";
68 }
69 return stream.str();
70 }
71
ColumnString(const Detail & detail,const int64_t cumulative_stat_on_node,const Stat<int64_t> & stat) const72 std::string StatsCalculator::ColumnString(const Detail& detail,
73 const int64_t cumulative_stat_on_node,
74 const Stat<int64_t>& stat) const {
75 const double start_ms = detail.start_us.avg() / 1000.0;
76 const double first_time_ms = detail.rel_end_us.first() / 1000.0;
77 const double avg_time_ms = detail.rel_end_us.avg() / 1000.0;
78 const double percentage = detail.rel_end_us.sum() * 100.0 / stat.sum();
79 const double cdf_percentage = (cumulative_stat_on_node * 100.0f) / stat.sum();
80 const int64_t times_called = detail.times_called / num_runs();
81
82 std::stringstream stream;
83 if (options_.format_as_csv) {
84 std::string name(detail.name);
85 std::replace(name.begin(), name.end(), ',', '\t');
86 stream << detail.type << ", " << start_ms << ", " << first_time_ms << ", "
87 << avg_time_ms << ", " << percentage << "%, " << cdf_percentage
88 << "%, " << detail.mem_used.newest() / 1000.0 << ", " << times_called
89 << ", " << name;
90 } else {
91 InitField(stream, 24) << detail.type;
92 InitField(stream, 17) << start_ms;
93 InitField(stream, 9) << first_time_ms;
94 InitField(stream, 9) << avg_time_ms;
95 InitField(stream, 7) << percentage << "%";
96 InitField(stream, 7) << cdf_percentage << "%";
97 InitField(stream, 10) << detail.mem_used.newest() / 1000.0;
98 InitField(stream, 9) << times_called;
99 stream << "\t" << detail.name;
100 }
101
102 return stream.str();
103 }
104
OrderNodesByMetric(SortingMetric metric,std::vector<const Detail * > * details) const105 void StatsCalculator::OrderNodesByMetric(
106 SortingMetric metric, std::vector<const Detail*>* details) const {
107 std::priority_queue<std::pair<std::string, const Detail*>> sorted_list;
108 const int num_nodes = details_.size();
109
110 for (const auto& det : details_) {
111 const Detail* detail = &(det.second);
112 std::stringstream stream;
113 stream << std::setw(20) << std::right << std::setprecision(10)
114 << std::fixed;
115
116 switch (metric) {
117 case BY_NAME:
118 stream << detail->name;
119 break;
120 case BY_RUN_ORDER:
121 stream << num_nodes - detail->run_order;
122 break;
123 case BY_TIME:
124 stream << detail->rel_end_us.avg();
125 break;
126 case BY_MEMORY:
127 stream << detail->mem_used.avg();
128 break;
129 case BY_TYPE:
130 stream << detail->type;
131 break;
132 default:
133 stream << "";
134 break;
135 }
136
137 sorted_list.emplace(stream.str(), detail);
138 }
139
140 while (!sorted_list.empty()) {
141 auto entry = sorted_list.top();
142 sorted_list.pop();
143 details->push_back(entry.second);
144 }
145 }
146
ComputeStatsByType(std::map<std::string,int64_t> * node_type_map_count,std::map<std::string,int64_t> * node_type_map_time,std::map<std::string,int64_t> * node_type_map_memory,std::map<std::string,int64_t> * node_type_map_times_called,int64_t * accumulated_us) const147 void StatsCalculator::ComputeStatsByType(
148 std::map<std::string, int64_t>* node_type_map_count,
149 std::map<std::string, int64_t>* node_type_map_time,
150 std::map<std::string, int64_t>* node_type_map_memory,
151 std::map<std::string, int64_t>* node_type_map_times_called,
152 int64_t* accumulated_us) const {
153 int64_t run_count = run_total_us_.count();
154
155 for (const auto& det : details_) {
156 const std::string node_name = det.first;
157 const Detail& detail = det.second;
158
159 int64_t curr_time_val =
160 static_cast<int64_t>(detail.rel_end_us.sum() / run_count);
161 *accumulated_us += curr_time_val;
162
163 int64_t curr_memory_val = detail.mem_used.newest();
164
165 const std::string& node_type = detail.type;
166
167 (*node_type_map_count)[node_type] += 1;
168 (*node_type_map_time)[node_type] += curr_time_val;
169 (*node_type_map_memory)[node_type] += curr_memory_val;
170 (*node_type_map_times_called)[node_type] += detail.times_called / run_count;
171 }
172 }
173
GetStatsByNodeType() const174 std::string StatsCalculator::GetStatsByNodeType() const {
175 std::stringstream stream;
176
177 stream << "Number of nodes executed: " << details_.size() << std::endl;
178
179 stream << "============================== Summary by node type "
180 "=============================="
181 << std::endl;
182
183 std::map<std::string, int64_t> node_type_map_count;
184 std::map<std::string, int64_t> node_type_map_time;
185 std::map<std::string, int64_t> node_type_map_memory;
186 std::map<std::string, int64_t> node_type_map_times_called;
187 int64_t accumulated_us = 0;
188
189 ComputeStatsByType(&node_type_map_count, &node_type_map_time,
190 &node_type_map_memory, &node_type_map_times_called,
191 &accumulated_us);
192
193 // Sort them.
194 std::priority_queue<std::pair<int64_t, std::pair<std::string, int64_t>>>
195 timings;
196 for (const auto& node_type : node_type_map_time) {
197 const int64_t mem_used = node_type_map_memory[node_type.first];
198 timings.emplace(node_type.second,
199 std::pair<std::string, int64_t>(node_type.first, mem_used));
200 }
201
202 if (options_.format_as_csv) {
203 stream << "node type, count, avg_ms, avg %, cdf %, mem KB, times called\n";
204 } else {
205 InitField(stream, 24) << "[Node type]";
206 InitField(stream, 9) << "[count]";
207 InitField(stream, 10) << "[avg ms]";
208 InitField(stream, 11) << "[avg %]";
209 InitField(stream, 11) << "[cdf %]";
210 InitField(stream, 10) << "[mem KB]";
211 InitField(stream, 10) << "[times called]";
212 stream << std::endl;
213 }
214
215 float cdf = 0.0f;
216 while (!timings.empty()) {
217 auto entry = timings.top();
218 timings.pop();
219
220 const std::string node_type = entry.second.first;
221 const float memory = entry.second.second / 1000.0f;
222
223 const int64_t node_type_total_us = entry.first;
224 const float time_per_run_ms = node_type_total_us / 1000.0f;
225
226 const float percentage =
227 ((entry.first / static_cast<float>(accumulated_us)) * 100.0f);
228 cdf += percentage;
229
230 if (options_.format_as_csv) {
231 stream << node_type << ", " << node_type_map_count[node_type] << ", "
232 << time_per_run_ms << ", " << percentage << "%, " << cdf << "%, "
233 << memory << ", " << node_type_map_times_called[node_type]
234 << std::endl;
235 } else {
236 InitField(stream, 24) << node_type;
237 InitField(stream, 9) << node_type_map_count[node_type];
238 InitField(stream, 10) << time_per_run_ms;
239 InitField(stream, 10) << percentage << "%";
240 InitField(stream, 10) << cdf << "%";
241 InitField(stream, 10) << memory;
242 InitField(stream, 9) << node_type_map_times_called[node_type];
243 stream << std::endl;
244 }
245 }
246 stream << std::endl;
247 return stream.str();
248 }
249
GetStatsByMetric(const std::string & title,SortingMetric sorting_metric,int num_stats) const250 std::string StatsCalculator::GetStatsByMetric(const std::string& title,
251 SortingMetric sorting_metric,
252 int num_stats) const {
253 std::vector<const Detail*> details;
254 OrderNodesByMetric(sorting_metric, &details);
255
256 double cumulative_stat_on_node = 0;
257
258 std::stringstream stream;
259 stream << HeaderString(title) << std::endl;
260 int stat_num = 0;
261 for (auto detail : details) {
262 ++stat_num;
263 if (num_stats > 0 && stat_num > num_stats) {
264 break;
265 }
266
267 // TODO(andrewharp): Make this keep track of the particular metric for cdf.
268 cumulative_stat_on_node += detail->rel_end_us.sum();
269 stream << ColumnString(*detail, cumulative_stat_on_node, run_total_us_)
270 << std::endl;
271 }
272 stream << std::endl;
273 return stream.str();
274 }
275
GetOutputString() const276 std::string StatsCalculator::GetOutputString() const {
277 std::stringstream stream;
278 if (options_.show_run_order) {
279 stream << GetStatsByMetric("Run Order", BY_RUN_ORDER,
280 options_.run_order_limit);
281 }
282 if (options_.show_time) {
283 stream << GetStatsByMetric("Top by Computation Time", BY_TIME,
284 options_.time_limit);
285 }
286 if (options_.show_memory) {
287 stream << GetStatsByMetric("Top by Memory Use", BY_MEMORY,
288 options_.memory_limit);
289 }
290 if (options_.show_type) {
291 stream << GetStatsByNodeType();
292 }
293 if (options_.show_summary) {
294 stream << GetShortSummary() << std::endl;
295 }
296 return stream.str();
297 }
298
AddNodeStats(const std::string & name,const std::string & type,int64_t run_order,int64_t start_us,int64_t rel_end_us,int64_t mem_used)299 void StatsCalculator::AddNodeStats(const std::string& name,
300 const std::string& type, int64_t run_order,
301 int64_t start_us, int64_t rel_end_us,
302 int64_t mem_used) {
303 Detail* detail = nullptr;
304 if (details_.find(name) == details_.end()) {
305 details_.insert({name, {}});
306 detail = &details_.at(name);
307 detail->type = type;
308 detail->name = name;
309 detail->run_order = run_order;
310 } else {
311 detail = &details_.at(name);
312 }
313 detail->start_us.UpdateStat(start_us);
314 detail->rel_end_us.UpdateStat(rel_end_us);
315 detail->mem_used.UpdateStat(mem_used);
316 detail->times_called++;
317 }
318
319 } // namespace tensorflow
320