1 /*
2 ** Copyright 2010 The Android Open Source Project
3 **
4 ** Licensed under the Apache License, Version 2.0 (the "License");
5 ** you may not use this file except in compliance with the License.
6 ** You may obtain a copy of the License at
7 **
8 ** http://www.apache.org/licenses/LICENSE-2.0
9 **
10 ** Unless required by applicable law or agreed to in writing, software
11 ** distributed under the License is distributed on an "AS IS" BASIS,
12 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 ** See the License for the specific language governing permissions and
14 ** limitations under the License.
15 */
16
17 /*
18 * Micro-benchmarking of sleep/cpu speed/memcpy/memset/memory reads/strcmp.
19 */
20
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <ctype.h>
25 #include <math.h>
26 #include <sched.h>
27 #include <sys/resource.h>
28 #include <time.h>
29 #include <unistd.h>
30
31 // The default size of data that will be manipulated in each iteration of
32 // a memory benchmark. Can be modified with the --data_size option.
33 #define DEFAULT_DATA_SIZE 1000000000
34
35 // The amount of memory allocated for the cold benchmarks to use.
36 #define DEFAULT_COLD_DATA_SIZE 128*1024*1024
37
38 // The default size of the stride between each buffer for cold benchmarks.
39 #define DEFAULT_COLD_STRIDE_SIZE 4096
40
41 // Number of nanoseconds in a second.
42 #define NS_PER_SEC 1000000000
43
44 // The maximum number of arguments that a benchmark will accept.
45 #define MAX_ARGS 2
46
47 // Default memory alignment of malloc.
48 #define DEFAULT_MALLOC_MEMORY_ALIGNMENT 8
49
50 // Contains information about benchmark options.
51 typedef struct {
52 bool print_average;
53 bool print_each_iter;
54
55 int dst_align;
56 int dst_or_mask;
57 int src_align;
58 int src_or_mask;
59
60 int cpu_to_lock;
61
62 int data_size;
63 int dst_str_size;
64 int cold_data_size;
65 int cold_stride_size;
66
67 int args[MAX_ARGS];
68 int num_args;
69 } command_data_t;
70
71 typedef void *(*void_func_t)();
72 typedef void *(*memcpy_func_t)(void *, const void *, size_t);
73 typedef void *(*memset_func_t)(void *, int, size_t);
74 typedef int (*strcmp_func_t)(const char *, const char *);
75 typedef char *(*str_func_t)(char *, const char *);
76 typedef size_t (*strlen_func_t)(const char *);
77
78 // Struct that contains a mapping of benchmark name to benchmark function.
79 typedef struct {
80 const char *name;
81 int (*ptr)(const char *, const command_data_t &, void_func_t func);
82 void_func_t func;
83 } function_t;
84
85 // Get the current time in nanoseconds.
nanoTime()86 uint64_t nanoTime() {
87 struct timespec t;
88
89 t.tv_sec = t.tv_nsec = 0;
90 clock_gettime(CLOCK_MONOTONIC, &t);
91 return static_cast<uint64_t>(t.tv_sec) * NS_PER_SEC + t.tv_nsec;
92 }
93
94 // Allocate memory with a specific alignment and return that pointer.
95 // This function assumes an alignment value that is a power of 2.
96 // If the alignment is 0, then use the pointer returned by malloc.
getAlignedMemory(uint8_t * orig_ptr,int alignment,int or_mask)97 uint8_t *getAlignedMemory(uint8_t *orig_ptr, int alignment, int or_mask) {
98 uint64_t ptr = reinterpret_cast<uint64_t>(orig_ptr);
99 if (alignment > 0) {
100 // When setting the alignment, set it to exactly the alignment chosen.
101 // The pointer returned will be guaranteed not to be aligned to anything
102 // more than that.
103 ptr += alignment - (ptr & (alignment - 1));
104 ptr |= alignment | or_mask;
105 }
106
107 return reinterpret_cast<uint8_t*>(ptr);
108 }
109
110 // Allocate memory with a specific alignment and return that pointer.
111 // This function assumes an alignment value that is a power of 2.
112 // If the alignment is 0, then use the pointer returned by malloc.
allocateAlignedMemory(size_t size,int alignment,int or_mask)113 uint8_t *allocateAlignedMemory(size_t size, int alignment, int or_mask) {
114 uint64_t ptr = reinterpret_cast<uint64_t>(malloc(size + 3 * alignment));
115 if (!ptr)
116 return NULL;
117 return getAlignedMemory((uint8_t*)ptr, alignment, or_mask);
118 }
119
initString(uint8_t * buf,size_t size)120 void initString(uint8_t *buf, size_t size) {
121 for (size_t i = 0; i < size - 1; i++) {
122 buf[i] = static_cast<char>(32 + (i % 96));
123 }
124 buf[size-1] = '\0';
125 }
126
computeAverage(uint64_t time_ns,size_t size,size_t copies)127 static inline double computeAverage(uint64_t time_ns, size_t size, size_t copies) {
128 return ((size/1024.0) * copies) / ((double)time_ns/NS_PER_SEC);
129 }
130
computeRunningAvg(double avg,double running_avg,size_t cur_idx)131 static inline double computeRunningAvg(double avg, double running_avg, size_t cur_idx) {
132 return (running_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1));
133 }
134
computeRunningSquareAvg(double avg,double square_avg,size_t cur_idx)135 static inline double computeRunningSquareAvg(double avg, double square_avg, size_t cur_idx) {
136 return (square_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)) * avg;
137 }
138
computeStdDev(double square_avg,double running_avg)139 static inline double computeStdDev(double square_avg, double running_avg) {
140 return sqrt(square_avg - running_avg * running_avg);
141 }
142
printIter(uint64_t time_ns,const char * name,size_t size,size_t copies,double avg)143 static inline void printIter(uint64_t time_ns, const char *name, size_t size, size_t copies, double avg) {
144 printf("%s %zux%zu bytes took %.06f seconds (%f MB/s)\n",
145 name, copies, size, (double)time_ns/NS_PER_SEC, avg/1024.0);
146 }
147
printSummary(uint64_t,const char * name,size_t size,size_t copies,double running_avg,double std_dev,double min,double max)148 static inline void printSummary(uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, double running_avg, double std_dev, double min, double max) {
149 printf(" %s %zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n",
150 name, copies, size, running_avg/1024.0, std_dev/1024.0, min/1024.0,
151 max/1024.0);
152 }
153
154 // For the cold benchmarks, a large buffer will be created which
155 // contains many "size" buffers. This function will figure out the increment
156 // needed between each buffer so that each one is aligned to "alignment".
getAlignmentIncrement(size_t size,int alignment)157 int getAlignmentIncrement(size_t size, int alignment) {
158 if (alignment == 0) {
159 alignment = DEFAULT_MALLOC_MEMORY_ALIGNMENT;
160 }
161 alignment *= 2;
162 return size + alignment - (size % alignment);
163 }
164
getColdBuffer(int num_buffers,size_t incr,int alignment,int or_mask)165 uint8_t *getColdBuffer(int num_buffers, size_t incr, int alignment, int or_mask) {
166 uint8_t *buffers = reinterpret_cast<uint8_t*>(malloc(num_buffers * incr + 3 * alignment));
167 if (!buffers) {
168 return NULL;
169 }
170 return getAlignedMemory(buffers, alignment, or_mask);
171 }
172
computeColdAverage(uint64_t time_ns,size_t size,size_t copies,size_t num_buffers)173 static inline double computeColdAverage(uint64_t time_ns, size_t size, size_t copies, size_t num_buffers) {
174 return ((size/1024.0) * copies * num_buffers) / ((double)time_ns/NS_PER_SEC);
175 }
176
printColdIter(uint64_t time_ns,const char * name,size_t size,size_t copies,size_t num_buffers,double avg)177 static void inline printColdIter(uint64_t time_ns, const char *name, size_t size, size_t copies, size_t num_buffers, double avg) {
178 printf("%s %zux%zux%zu bytes took %.06f seconds (%f MB/s)\n",
179 name, copies, num_buffers, size, (double)time_ns/NS_PER_SEC, avg/1024.0);
180 }
181
printColdSummary(uint64_t,const char * name,size_t size,size_t copies,size_t num_buffers,double running_avg,double square_avg,double min,double max)182 static void inline printColdSummary(
183 uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, size_t num_buffers,
184 double running_avg, double square_avg, double min, double max) {
185 printf(" %s %zux%zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n",
186 name, copies, num_buffers, size, running_avg/1024.0,
187 computeStdDev(running_avg, square_avg)/1024.0, min/1024.0, max/1024.0);
188 }
189
190 #define MAINLOOP(cmd_data, BENCH, COMPUTE_AVG, PRINT_ITER, PRINT_AVG) \
191 uint64_t time_ns; \
192 int iters = cmd_data.args[1]; \
193 bool print_average = cmd_data.print_average; \
194 bool print_each_iter = cmd_data.print_each_iter; \
195 double min = 0.0, max = 0.0, running_avg = 0.0, square_avg = 0.0; \
196 double avg; \
197 for (int i = 0; iters == -1 || i < iters; i++) { \
198 time_ns = nanoTime(); \
199 BENCH; \
200 time_ns = nanoTime() - time_ns; \
201 avg = COMPUTE_AVG; \
202 if (print_average) { \
203 running_avg = computeRunningAvg(avg, running_avg, i); \
204 square_avg = computeRunningSquareAvg(avg, square_avg, i); \
205 if (min == 0.0 || avg < min) { \
206 min = avg; \
207 } \
208 if (avg > max) { \
209 max = avg; \
210 } \
211 } \
212 if (print_each_iter) { \
213 PRINT_ITER; \
214 } \
215 } \
216 if (print_average) { \
217 PRINT_AVG; \
218 }
219
220 #define MAINLOOP_DATA(name, cmd_data, size, BENCH) \
221 size_t copies = cmd_data.data_size/size; \
222 size_t j; \
223 MAINLOOP(cmd_data, \
224 for (j = 0; j < copies; j++) { \
225 BENCH; \
226 }, \
227 computeAverage(time_ns, size, copies), \
228 printIter(time_ns, name, size, copies, avg), \
229 double std_dev = computeStdDev(square_avg, running_avg); \
230 printSummary(time_ns, name, size, copies, running_avg, \
231 std_dev, min, max));
232
233 #define MAINLOOP_COLD(name, cmd_data, size, num_incrs, BENCH) \
234 size_t num_strides = num_buffers / num_incrs; \
235 if ((num_buffers % num_incrs) != 0) { \
236 num_strides--; \
237 } \
238 size_t copies = 1; \
239 num_buffers = num_incrs * num_strides; \
240 if (num_buffers * size < static_cast<size_t>(cmd_data.data_size)) { \
241 copies = cmd_data.data_size / (num_buffers * size); \
242 } \
243 if (num_strides == 0) { \
244 printf("%s: Chosen options lead to no copies, aborting.\n", name); \
245 return -1; \
246 } \
247 size_t j, k; \
248 MAINLOOP(cmd_data, \
249 for (j = 0; j < copies; j++) { \
250 for (k = 0; k < num_incrs; k++) { \
251 BENCH; \
252 } \
253 }, \
254 computeColdAverage(time_ns, size, copies, num_buffers), \
255 printColdIter(time_ns, name, size, copies, num_buffers, avg), \
256 printColdSummary(time_ns, name, size, copies, num_buffers, \
257 running_avg, square_avg, min, max));
258
259 // This version of the macro creates a single buffer of the given size and
260 // alignment. The variable "buf" will be a pointer to the buffer and should
261 // be used by the BENCH code.
262 // INIT - Any specialized code needed to initialize the data. This will only
263 // be executed once.
264 // BENCH - The actual code to benchmark and is timed.
265 #define BENCH_ONE_BUF(name, cmd_data, INIT, BENCH) \
266 size_t size = cmd_data.args[0]; \
267 uint8_t *buf = allocateAlignedMemory(size, cmd_data.dst_align, cmd_data.dst_or_mask); \
268 if (!buf) \
269 return -1; \
270 INIT; \
271 MAINLOOP_DATA(name, cmd_data, size, BENCH);
272
273 // This version of the macro creates two buffers of the given sizes and
274 // alignments. The variables "buf1" and "buf2" will be pointers to the
275 // buffers and should be used by the BENCH code.
276 // INIT - Any specialized code needed to initialize the data. This will only
277 // be executed once.
278 // BENCH - The actual code to benchmark and is timed.
279 #define BENCH_TWO_BUFS(name, cmd_data, INIT, BENCH) \
280 size_t size = cmd_data.args[0]; \
281 uint8_t *buf1 = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask); \
282 if (!buf1) \
283 return -1; \
284 size_t total_size = size; \
285 if (cmd_data.dst_str_size > 0) \
286 total_size += cmd_data.dst_str_size; \
287 uint8_t *buf2 = allocateAlignedMemory(total_size, cmd_data.dst_align, cmd_data.dst_or_mask); \
288 if (!buf2) \
289 return -1; \
290 INIT; \
291 MAINLOOP_DATA(name, cmd_data, size, BENCH);
292
293 // This version of the macro attempts to benchmark code when the data
294 // being manipulated is not in the cache, thus the cache is cold. It does
295 // this by creating a single large buffer that is designed to be larger than
296 // the largest cache in the system. The variable "buf" will be one slice
297 // of the buffer that the BENCH code should use that is of the correct size
298 // and alignment. In order to avoid any algorithms that prefetch past the end
299 // of their "buf" and into the next sequential buffer, the code strides
300 // through the buffer. Specifically, as "buf" values are iterated in BENCH
301 // code, the end of "buf" is guaranteed to be at least "stride_size" away
302 // from the next "buf".
303 // INIT - Any specialized code needed to initialize the data. This will only
304 // be executed once.
305 // BENCH - The actual code to benchmark and is timed.
306 #define COLD_ONE_BUF(name, cmd_data, INIT, BENCH) \
307 size_t size = cmd_data.args[0]; \
308 size_t incr = getAlignmentIncrement(size, cmd_data.dst_align); \
309 size_t num_buffers = cmd_data.cold_data_size / incr; \
310 size_t buffer_size = num_buffers * incr; \
311 uint8_t *buffer = getColdBuffer(num_buffers, incr, cmd_data.dst_align, cmd_data.dst_or_mask); \
312 if (!buffer) \
313 return -1; \
314 size_t num_incrs = cmd_data.cold_stride_size / incr + 1; \
315 size_t stride_incr = incr * num_incrs; \
316 uint8_t *buf; \
317 size_t l; \
318 INIT; \
319 MAINLOOP_COLD(name, cmd_data, size, num_incrs, \
320 buf = buffer + k * incr; \
321 for (l = 0; l < num_strides; l++) { \
322 BENCH; \
323 buf += stride_incr; \
324 });
325
326 // This version of the macro attempts to benchmark code when the data
327 // being manipulated is not in the cache, thus the cache is cold. It does
328 // this by creating two large buffers each of which is designed to be
329 // larger than the largest cache in the system. Two variables "buf1" and
330 // "buf2" will be the two buffers that BENCH code should use. In order
331 // to avoid any algorithms that prefetch past the end of either "buf1"
332 // or "buf2" and into the next sequential buffer, the code strides through
333 // both buffers. Specifically, as "buf1" and "buf2" values are iterated in
334 // BENCH code, the end of "buf1" and "buf2" is guaranteed to be at least
335 // "stride_size" away from the next "buf1" and "buf2".
336 // INIT - Any specialized code needed to initialize the data. This will only
337 // be executed once.
338 // BENCH - The actual code to benchmark and is timed.
339 #define COLD_TWO_BUFS(name, cmd_data, INIT, BENCH) \
340 size_t size = cmd_data.args[0]; \
341 size_t buf1_incr = getAlignmentIncrement(size, cmd_data.src_align); \
342 size_t total_size = size; \
343 if (cmd_data.dst_str_size > 0) \
344 total_size += cmd_data.dst_str_size; \
345 size_t buf2_incr = getAlignmentIncrement(total_size, cmd_data.dst_align); \
346 size_t max_incr = (buf1_incr > buf2_incr) ? buf1_incr : buf2_incr; \
347 size_t num_buffers = cmd_data.cold_data_size / max_incr; \
348 size_t buffer1_size = num_buffers * buf1_incr; \
349 size_t buffer2_size = num_buffers * buf2_incr; \
350 uint8_t *buffer1 = getColdBuffer(num_buffers, buf1_incr, cmd_data.src_align, cmd_data.src_or_mask); \
351 if (!buffer1) \
352 return -1; \
353 uint8_t *buffer2 = getColdBuffer(num_buffers, buf2_incr, cmd_data.dst_align, cmd_data.dst_or_mask); \
354 if (!buffer2) \
355 return -1; \
356 size_t min_incr = (buf1_incr < buf2_incr) ? buf1_incr : buf2_incr; \
357 size_t num_incrs = cmd_data.cold_stride_size / min_incr + 1; \
358 size_t buf1_stride_incr = buf1_incr * num_incrs; \
359 size_t buf2_stride_incr = buf2_incr * num_incrs; \
360 size_t l; \
361 uint8_t *buf1; \
362 uint8_t *buf2; \
363 INIT; \
364 MAINLOOP_COLD(name, cmd_data, size, num_incrs, \
365 buf1 = buffer1 + k * buf1_incr; \
366 buf2 = buffer2 + k * buf2_incr; \
367 for (l = 0; l < num_strides; l++) { \
368 BENCH; \
369 buf1 += buf1_stride_incr; \
370 buf2 += buf2_stride_incr; \
371 });
372
benchmarkSleep(const char *,const command_data_t & cmd_data,void_func_t)373 int benchmarkSleep(const char* /*name*/, const command_data_t &cmd_data, void_func_t /*func*/) {
374 int delay = cmd_data.args[0];
375 MAINLOOP(cmd_data, sleep(delay),
376 (double)time_ns/NS_PER_SEC,
377 printf("sleep(%d) took %.06f seconds\n", delay, avg);,
378 printf(" sleep(%d) average %.06f seconds std dev %f min %.06f seconds max %0.6f seconds\n", \
379 delay, running_avg, computeStdDev(square_avg, running_avg), \
380 min, max));
381
382 return 0;
383 }
384
benchmarkCpu(const char *,const command_data_t & cmd_data,void_func_t)385 int benchmarkCpu(const char* /*name*/, const command_data_t &cmd_data, void_func_t /*func*/) {
386 // Use volatile so that the loop is not optimized away by the compiler.
387 volatile int cpu_foo;
388
389 MAINLOOP(cmd_data,
390 for (cpu_foo = 0; cpu_foo < 100000000; cpu_foo++),
391 (double)time_ns/NS_PER_SEC,
392 printf("cpu took %.06f seconds\n", avg),
393 printf(" cpu average %.06f seconds std dev %f min %0.6f seconds max %0.6f seconds\n", \
394 running_avg, computeStdDev(square_avg, running_avg), min, max));
395
396 return 0;
397 }
398
benchmarkMemset(const char * name,const command_data_t & cmd_data,void_func_t func)399 int benchmarkMemset(const char *name, const command_data_t &cmd_data, void_func_t func) {
400 memset_func_t memset_func = reinterpret_cast<memset_func_t>(func);
401 BENCH_ONE_BUF(name, cmd_data, ;, memset_func(buf, i, size));
402
403 return 0;
404 }
405
benchmarkMemsetCold(const char * name,const command_data_t & cmd_data,void_func_t func)406 int benchmarkMemsetCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
407 memset_func_t memset_func = reinterpret_cast<memset_func_t>(func);
408 COLD_ONE_BUF(name, cmd_data, ;, memset_func(buf, l, size));
409
410 return 0;
411 }
412
benchmarkMemcpy(const char * name,const command_data_t & cmd_data,void_func_t func)413 int benchmarkMemcpy(const char *name, const command_data_t &cmd_data, void_func_t func) {
414 memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func);
415
416 BENCH_TWO_BUFS(name, cmd_data,
417 memset(buf1, 0xff, size); \
418 memset(buf2, 0, size),
419 memcpy_func(buf2, buf1, size));
420
421 return 0;
422 }
423
benchmarkMemcpyCold(const char * name,const command_data_t & cmd_data,void_func_t func)424 int benchmarkMemcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
425 memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func);
426
427 COLD_TWO_BUFS(name, cmd_data,
428 memset(buffer1, 0xff, buffer1_size); \
429 memset(buffer2, 0x0, buffer2_size),
430 memcpy_func(buf2, buf1, size));
431
432 return 0;
433 }
434
benchmarkMemmoveBackwards(const char * name,const command_data_t & cmd_data,void_func_t func)435 int benchmarkMemmoveBackwards(const char *name, const command_data_t &cmd_data, void_func_t func) {
436 memcpy_func_t memmove_func = reinterpret_cast<memcpy_func_t>(func);
437
438 size_t size = cmd_data.args[0];
439 size_t alloc_size = size * 2 + 3 * cmd_data.dst_align;
440 uint8_t* src = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask);
441 if (!src)
442 return -1;
443 // Force memmove to do a backwards copy by getting a pointer into the source buffer.
444 uint8_t* dst = getAlignedMemory(src+1, cmd_data.dst_align, cmd_data.dst_or_mask);
445 if (!dst)
446 return -1;
447 MAINLOOP_DATA(name, cmd_data, size, memmove_func(dst, src, size));
448 return 0;
449 }
450
benchmarkMemread(const char * name,const command_data_t & cmd_data,void_func_t)451 int benchmarkMemread(const char *name, const command_data_t &cmd_data, void_func_t /*func*/) {
452 int size = cmd_data.args[0];
453
454 uint32_t *src = reinterpret_cast<uint32_t*>(malloc(size));
455 if (!src)
456 return -1;
457 memset(src, 0xff, size);
458
459 // Use volatile so the compiler does not optimize away the reads.
460 volatile int foo;
461 size_t k;
462 MAINLOOP_DATA(name, cmd_data, size,
463 for (k = 0; k < size/sizeof(uint32_t); k++) foo = src[k]);
464
465 return 0;
466 }
467
benchmarkStrcmp(const char * name,const command_data_t & cmd_data,void_func_t func)468 int benchmarkStrcmp(const char *name, const command_data_t &cmd_data, void_func_t func) {
469 strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func);
470
471 int retval;
472 BENCH_TWO_BUFS(name, cmd_data,
473 initString(buf1, size); \
474 initString(buf2, size),
475 retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \
476 if (retval != 0) printf("%s failed, return value %d\n", name, retval));
477
478 return 0;
479 }
480
benchmarkStrcmpCold(const char * name,const command_data_t & cmd_data,void_func_t func)481 int benchmarkStrcmpCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
482 strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func);
483
484 int retval;
485 COLD_TWO_BUFS(name, cmd_data,
486 memset(buffer1, 'a', buffer1_size); \
487 memset(buffer2, 'a', buffer2_size); \
488 for (size_t i =0; i < num_buffers; i++) { \
489 buffer1[size-1+buf1_incr*i] = '\0'; \
490 buffer2[size-1+buf2_incr*i] = '\0'; \
491 },
492 retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \
493 if (retval != 0) printf("%s failed, return value %d\n", name, retval));
494
495 return 0;
496 }
497
benchmarkStrlen(const char * name,const command_data_t & cmd_data,void_func_t func)498 int benchmarkStrlen(const char *name, const command_data_t &cmd_data, void_func_t func) {
499 size_t real_size;
500 strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func);
501 BENCH_ONE_BUF(name, cmd_data,
502 initString(buf, size),
503 real_size = strlen_func(reinterpret_cast<char*>(buf)); \
504 if (real_size + 1 != size) { \
505 printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \
506 return -1; \
507 });
508
509 return 0;
510 }
511
benchmarkStrlenCold(const char * name,const command_data_t & cmd_data,void_func_t func)512 int benchmarkStrlenCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
513 strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func);
514 size_t real_size;
515 COLD_ONE_BUF(name, cmd_data,
516 memset(buffer, 'a', buffer_size); \
517 for (size_t i = 0; i < num_buffers; i++) { \
518 buffer[size-1+incr*i] = '\0'; \
519 },
520 real_size = strlen_func(reinterpret_cast<char*>(buf)); \
521 if (real_size + 1 != size) { \
522 printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \
523 return -1; \
524 });
525 return 0;
526 }
527
benchmarkStrcat(const char * name,const command_data_t & cmd_data,void_func_t func)528 int benchmarkStrcat(const char *name, const command_data_t &cmd_data, void_func_t func) {
529 str_func_t str_func = reinterpret_cast<str_func_t>(func);
530
531 int dst_str_size = cmd_data.dst_str_size;
532 if (dst_str_size <= 0) {
533 printf("%s requires --dst_str_size to be set to a non-zero value.\n",
534 name);
535 return -1;
536 }
537 BENCH_TWO_BUFS(name, cmd_data,
538 initString(buf1, size); \
539 initString(buf2, dst_str_size),
540 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0');
541
542 return 0;
543 }
544
benchmarkStrcatCold(const char * name,const command_data_t & cmd_data,void_func_t func)545 int benchmarkStrcatCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
546 str_func_t str_func = reinterpret_cast<str_func_t>(func);
547
548 int dst_str_size = cmd_data.dst_str_size;
549 if (dst_str_size <= 0) {
550 printf("%s requires --dst_str_size to be set to a non-zero value.\n",
551 name);
552 return -1;
553 }
554 COLD_TWO_BUFS(name, cmd_data,
555 memset(buffer1, 'a', buffer1_size); \
556 memset(buffer2, 'b', buffer2_size); \
557 for (size_t i = 0; i < num_buffers; i++) { \
558 buffer1[size-1+buf1_incr*i] = '\0'; \
559 buffer2[dst_str_size-1+buf2_incr*i] = '\0'; \
560 },
561 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0');
562
563 return 0;
564 }
565
566
benchmarkStrcpy(const char * name,const command_data_t & cmd_data,void_func_t func)567 int benchmarkStrcpy(const char *name, const command_data_t &cmd_data, void_func_t func) {
568 str_func_t str_func = reinterpret_cast<str_func_t>(func);
569
570 BENCH_TWO_BUFS(name, cmd_data,
571 initString(buf1, size); \
572 memset(buf2, 0, size),
573 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)));
574
575 return 0;
576 }
577
benchmarkStrcpyCold(const char * name,const command_data_t & cmd_data,void_func_t func)578 int benchmarkStrcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
579 str_func_t str_func = reinterpret_cast<str_func_t>(func);
580
581 COLD_TWO_BUFS(name, cmd_data,
582 memset(buffer1, 'a', buffer1_size); \
583 for (size_t i = 0; i < num_buffers; i++) { \
584 buffer1[size-1+buf1_incr*i] = '\0'; \
585 } \
586 memset(buffer2, 0, buffer2_size),
587 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)));
588
589 return 0;
590 }
591
592 // Create the mapping structure.
593 function_t function_table[] = {
594 { "cpu", benchmarkCpu, NULL },
595 { "memcpy", benchmarkMemcpy, reinterpret_cast<void_func_t>(memcpy) },
596 { "memcpy_cold", benchmarkMemcpyCold, reinterpret_cast<void_func_t>(memcpy) },
597 { "memmove_forward", benchmarkMemcpy, reinterpret_cast<void_func_t>(memmove) },
598 { "memmove_backward", benchmarkMemmoveBackwards, reinterpret_cast<void_func_t>(memmove) },
599 { "memread", benchmarkMemread, NULL },
600 { "memset", benchmarkMemset, reinterpret_cast<void_func_t>(memset) },
601 { "memset_cold", benchmarkMemsetCold, reinterpret_cast<void_func_t>(memset) },
602 { "sleep", benchmarkSleep, NULL },
603 { "strcat", benchmarkStrcat, reinterpret_cast<void_func_t>(strcat) },
604 { "strcat_cold", benchmarkStrcatCold, reinterpret_cast<void_func_t>(strcat) },
605 { "strcmp", benchmarkStrcmp, reinterpret_cast<void_func_t>(strcmp) },
606 { "strcmp_cold", benchmarkStrcmpCold, reinterpret_cast<void_func_t>(strcmp) },
607 { "strcpy", benchmarkStrcpy, reinterpret_cast<void_func_t>(strcpy) },
608 { "strcpy_cold", benchmarkStrcpyCold, reinterpret_cast<void_func_t>(strcpy) },
609 { "strlen", benchmarkStrlen, reinterpret_cast<void_func_t>(strlen) },
610 { "strlen_cold", benchmarkStrlenCold, reinterpret_cast<void_func_t>(strlen) },
611 };
612
usage()613 void usage() {
614 printf("Usage:\n");
615 printf(" micro_bench [--data_size DATA_BYTES] [--print_average]\n");
616 printf(" [--no_print_each_iter] [--lock_to_cpu CORE]\n");
617 printf(" [--src_align ALIGN] [--src_or_mask OR_MASK]\n");
618 printf(" [--dst_align ALIGN] [--dst_or_mask OR_MASK]\n");
619 printf(" [--dst_str_size SIZE] [--cold_data_size DATA_BYTES]\n");
620 printf(" [--cold_stride_size SIZE]\n");
621 printf(" --data_size DATA_BYTES\n");
622 printf(" For the data benchmarks (memcpy/memset/memread) the approximate\n");
623 printf(" size of data, in bytes, that will be manipulated in each iteration.\n");
624 printf(" --print_average\n");
625 printf(" Print the average and standard deviation of all iterations.\n");
626 printf(" --no_print_each_iter\n");
627 printf(" Do not print any values in each iteration.\n");
628 printf(" --lock_to_cpu CORE\n");
629 printf(" Lock to the specified CORE. The default is to use the last core found.\n");
630 printf(" --dst_align ALIGN\n");
631 printf(" If the command supports it, align the destination pointer to ALIGN.\n");
632 printf(" The default is to use the value returned by malloc.\n");
633 printf(" --dst_or_mask OR_MASK\n");
634 printf(" If the command supports it, or in the OR_MASK on to the destination pointer.\n");
635 printf(" The OR_MASK must be smaller than the dst_align value.\n");
636 printf(" The default value is 0.\n");
637
638 printf(" --src_align ALIGN\n");
639 printf(" If the command supports it, align the source pointer to ALIGN. The default is to use the\n");
640 printf(" value returned by malloc.\n");
641 printf(" --src_or_mask OR_MASK\n");
642 printf(" If the command supports it, or in the OR_MASK on to the source pointer.\n");
643 printf(" The OR_MASK must be smaller than the src_align value.\n");
644 printf(" The default value is 0.\n");
645 printf(" --dst_str_size SIZE\n");
646 printf(" If the command supports it, create a destination string of this length.\n");
647 printf(" The default is to not update the destination string.\n");
648 printf(" --cold_data_size DATA_SIZE\n");
649 printf(" For _cold benchmarks, use this as the total amount of memory to use.\n");
650 printf(" The default is 128MB, and the number should be larger than the cache on the chip.\n");
651 printf(" This value is specified in bytes.\n");
652 printf(" --cold_stride_size SIZE\n");
653 printf(" For _cold benchmarks, use this as the minimum stride between iterations.\n");
654 printf(" The default is 4096 bytes and the number should be larger than the amount of data\n");
655 printf(" pulled in to the cache by each run of the benchmark.\n");
656 printf(" ITERS\n");
657 printf(" The number of iterations to execute each benchmark. If not\n");
658 printf(" passed in then run forever.\n");
659 printf(" micro_bench cpu UNUSED [ITERS]\n");
660 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memcpy NUM_BYTES [ITERS]\n");
661 printf(" micro_bench memread NUM_BYTES [ITERS]\n");
662 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memset NUM_BYTES [ITERS]\n");
663 printf(" micro_bench sleep TIME_TO_SLEEP [ITERS]\n");
664 printf(" TIME_TO_SLEEP\n");
665 printf(" The time in seconds to sleep.\n");
666 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] [--dst_str_size SIZE] strcat NUM_BYTES [ITERS]\n");
667 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask OR_MASK] strcmp NUM_BYTES [ITERS]\n");
668 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] strcpy NUM_BYTES [ITERS]\n");
669 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] strlen NUM_BYTES [ITERS]\n");
670 printf("\n");
671 printf(" In addition, memcpy/memcpy/memset/strcat/strcpy/strlen have _cold versions\n");
672 printf(" that will execute the function on a buffer not in the cache.\n");
673 }
674
processOptions(int argc,char ** argv,command_data_t * cmd_data)675 function_t *processOptions(int argc, char **argv, command_data_t *cmd_data) {
676 function_t *command = NULL;
677
678 // Initialize the command_flags.
679 cmd_data->print_average = false;
680 cmd_data->print_each_iter = true;
681 cmd_data->dst_align = 0;
682 cmd_data->src_align = 0;
683 cmd_data->src_or_mask = 0;
684 cmd_data->dst_or_mask = 0;
685 cmd_data->num_args = 0;
686 cmd_data->cpu_to_lock = -1;
687 cmd_data->data_size = DEFAULT_DATA_SIZE;
688 cmd_data->dst_str_size = -1;
689 cmd_data->cold_data_size = DEFAULT_COLD_DATA_SIZE;
690 cmd_data->cold_stride_size = DEFAULT_COLD_STRIDE_SIZE;
691 for (int i = 0; i < MAX_ARGS; i++) {
692 cmd_data->args[i] = -1;
693 }
694
695 for (int i = 1; i < argc; i++) {
696 if (argv[i][0] == '-') {
697 int *save_value = NULL;
698 if (strcmp(argv[i], "--print_average") == 0) {
699 cmd_data->print_average = true;
700 } else if (strcmp(argv[i], "--no_print_each_iter") == 0) {
701 cmd_data->print_each_iter = false;
702 } else if (strcmp(argv[i], "--dst_align") == 0) {
703 save_value = &cmd_data->dst_align;
704 } else if (strcmp(argv[i], "--src_align") == 0) {
705 save_value = &cmd_data->src_align;
706 } else if (strcmp(argv[i], "--dst_or_mask") == 0) {
707 save_value = &cmd_data->dst_or_mask;
708 } else if (strcmp(argv[i], "--src_or_mask") == 0) {
709 save_value = &cmd_data->src_or_mask;
710 } else if (strcmp(argv[i], "--lock_to_cpu") == 0) {
711 save_value = &cmd_data->cpu_to_lock;
712 } else if (strcmp(argv[i], "--data_size") == 0) {
713 save_value = &cmd_data->data_size;
714 } else if (strcmp(argv[i], "--dst_str_size") == 0) {
715 save_value = &cmd_data->dst_str_size;
716 } else if (strcmp(argv[i], "--cold_data_size") == 0) {
717 save_value = &cmd_data->cold_data_size;
718 } else if (strcmp(argv[i], "--cold_stride_size") == 0) {
719 save_value = &cmd_data->cold_stride_size;
720 } else {
721 printf("Unknown option %s\n", argv[i]);
722 return NULL;
723 }
724 if (save_value) {
725 // Checking both characters without a strlen() call should be
726 // safe since as long as the argument exists, one character will
727 // be present (\0). And if the first character is '-', then
728 // there will always be a second character (\0 again).
729 if (i == argc - 1 || (argv[i + 1][0] == '-' && !isdigit(argv[i + 1][1]))) {
730 printf("The option %s requires one argument.\n",
731 argv[i]);
732 return NULL;
733 }
734 *save_value = (int)strtol(argv[++i], NULL, 0);
735 }
736 } else if (!command) {
737 for (size_t j = 0; j < sizeof(function_table)/sizeof(function_t); j++) {
738 if (strcmp(argv[i], function_table[j].name) == 0) {
739 command = &function_table[j];
740 break;
741 }
742 }
743 if (!command) {
744 printf("Uknown command %s\n", argv[i]);
745 return NULL;
746 }
747 } else if (cmd_data->num_args > MAX_ARGS) {
748 printf("More than %d number arguments passed in.\n", MAX_ARGS);
749 return NULL;
750 } else {
751 cmd_data->args[cmd_data->num_args++] = atoi(argv[i]);
752 }
753 }
754
755 // Check the arguments passed in make sense.
756 if (cmd_data->num_args != 1 && cmd_data->num_args != 2) {
757 printf("Not enough arguments passed in.\n");
758 return NULL;
759 } else if (cmd_data->dst_align < 0) {
760 printf("The --dst_align option must be greater than or equal to 0.\n");
761 return NULL;
762 } else if (cmd_data->src_align < 0) {
763 printf("The --src_align option must be greater than or equal to 0.\n");
764 return NULL;
765 } else if (cmd_data->data_size <= 0) {
766 printf("The --data_size option must be a positive number.\n");
767 return NULL;
768 } else if ((cmd_data->dst_align & (cmd_data->dst_align - 1))) {
769 printf("The --dst_align option must be a power of 2.\n");
770 return NULL;
771 } else if ((cmd_data->src_align & (cmd_data->src_align - 1))) {
772 printf("The --src_align option must be a power of 2.\n");
773 return NULL;
774 } else if (!cmd_data->src_align && cmd_data->src_or_mask) {
775 printf("The --src_or_mask option requires that --src_align be set.\n");
776 return NULL;
777 } else if (!cmd_data->dst_align && cmd_data->dst_or_mask) {
778 printf("The --dst_or_mask option requires that --dst_align be set.\n");
779 return NULL;
780 } else if (cmd_data->src_or_mask > cmd_data->src_align) {
781 printf("The value of --src_or_mask cannot be larger that --src_align.\n");
782 return NULL;
783 } else if (cmd_data->dst_or_mask > cmd_data->dst_align) {
784 printf("The value of --src_or_mask cannot be larger that --src_align.\n");
785 return NULL;
786 }
787
788 return command;
789 }
790
raisePriorityAndLock(int cpu_to_lock)791 bool raisePriorityAndLock(int cpu_to_lock) {
792 cpu_set_t cpuset;
793
794 if (setpriority(PRIO_PROCESS, 0, -20)) {
795 perror("Unable to raise priority of process.\n");
796 return false;
797 }
798
799 CPU_ZERO(&cpuset);
800 if (sched_getaffinity(0, sizeof(cpuset), &cpuset) != 0) {
801 perror("sched_getaffinity failed");
802 return false;
803 }
804
805 if (cpu_to_lock < 0) {
806 // Lock to the last active core we find.
807 for (int i = 0; i < CPU_SETSIZE; i++) {
808 if (CPU_ISSET(i, &cpuset)) {
809 cpu_to_lock = i;
810 }
811 }
812 } else if (!CPU_ISSET(cpu_to_lock, &cpuset)) {
813 printf("Cpu %d does not exist.\n", cpu_to_lock);
814 return false;
815 }
816
817 if (cpu_to_lock < 0) {
818 printf("Cannot find any valid cpu to lock.\n");
819 return false;
820 }
821
822 CPU_ZERO(&cpuset);
823 CPU_SET(cpu_to_lock, &cpuset);
824 if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) {
825 perror("sched_setaffinity failed");
826 return false;
827 }
828
829 return true;
830 }
831
main(int argc,char ** argv)832 int main(int argc, char **argv) {
833 command_data_t cmd_data;
834
835 function_t *command = processOptions(argc, argv, &cmd_data);
836 if (!command) {
837 usage();
838 return -1;
839 }
840
841 if (!raisePriorityAndLock(cmd_data.cpu_to_lock)) {
842 return -1;
843 }
844
845 printf("%s\n", command->name);
846 return (*command->ptr)(command->name, cmd_data, command->func);
847 }
848