1/* 2** Copyright 2010 The Android Open Source Project 3** 4** Licensed under the Apache License, Version 2.0 (the "License"); 5** you may not use this file except in compliance with the License. 6** You may obtain a copy of the License at 7** 8** http://www.apache.org/licenses/LICENSE-2.0 9** 10** Unless required by applicable law or agreed to in writing, software 11** distributed under the License is distributed on an "AS IS" BASIS, 12** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13** See the License for the specific language governing permissions and 14** limitations under the License. 15*/ 16 17/* 18 * Micro-benchmarking of sleep/cpu speed/memcpy/memset/memory reads/strcmp. 19 */ 20 21#include <stdio.h> 22#include <stdlib.h> 23#include <ctype.h> 24#include <math.h> 25#include <sched.h> 26#include <sys/resource.h> 27#include <time.h> 28#include <unistd.h> 29 30// The default size of data that will be manipulated in each iteration of 31// a memory benchmark. Can be modified with the --data_size option. 32#define DEFAULT_DATA_SIZE 1000000000 33 34// The amount of memory allocated for the cold benchmarks to use. 35#define DEFAULT_COLD_DATA_SIZE 128*1024*1024 36 37// The default size of the stride between each buffer for cold benchmarks. 38#define DEFAULT_COLD_STRIDE_SIZE 4096 39 40// Number of nanoseconds in a second. 41#define NS_PER_SEC 1000000000 42 43// The maximum number of arguments that a benchmark will accept. 44#define MAX_ARGS 2 45 46// Default memory alignment of malloc. 47#define DEFAULT_MALLOC_MEMORY_ALIGNMENT 8 48 49// Contains information about benchmark options. 50typedef struct { 51 bool print_average; 52 bool print_each_iter; 53 54 int dst_align; 55 int dst_or_mask; 56 int src_align; 57 int src_or_mask; 58 59 int cpu_to_lock; 60 61 int data_size; 62 int dst_str_size; 63 int cold_data_size; 64 int cold_stride_size; 65 66 int args[MAX_ARGS]; 67 int num_args; 68} command_data_t; 69 70typedef void *(*void_func_t)(); 71typedef void *(*memcpy_func_t)(void *, const void *, size_t); 72typedef void *(*memset_func_t)(void *, int, size_t); 73typedef int (*strcmp_func_t)(const char *, const char *); 74typedef char *(*str_func_t)(char *, const char *); 75typedef size_t (*strlen_func_t)(const char *); 76 77// Struct that contains a mapping of benchmark name to benchmark function. 78typedef struct { 79 const char *name; 80 int (*ptr)(const char *, const command_data_t &, void_func_t func); 81 void_func_t func; 82} function_t; 83 84// Get the current time in nanoseconds. 85uint64_t nanoTime() { 86 struct timespec t; 87 88 t.tv_sec = t.tv_nsec = 0; 89 clock_gettime(CLOCK_MONOTONIC, &t); 90 return static_cast<uint64_t>(t.tv_sec) * NS_PER_SEC + t.tv_nsec; 91} 92 93// Allocate memory with a specific alignment and return that pointer. 94// This function assumes an alignment value that is a power of 2. 95// If the alignment is 0, then use the pointer returned by malloc. 96uint8_t *getAlignedMemory(uint8_t *orig_ptr, int alignment, int or_mask) { 97 uint64_t ptr = reinterpret_cast<uint64_t>(orig_ptr); 98 if (alignment > 0) { 99 // When setting the alignment, set it to exactly the alignment chosen. 100 // The pointer returned will be guaranteed not to be aligned to anything 101 // more than that. 102 ptr += alignment - (ptr & (alignment - 1)); 103 ptr |= alignment | or_mask; 104 } 105 106 return reinterpret_cast<uint8_t*>(ptr); 107} 108 109// Allocate memory with a specific alignment and return that pointer. 110// This function assumes an alignment value that is a power of 2. 111// If the alignment is 0, then use the pointer returned by malloc. 112uint8_t *allocateAlignedMemory(size_t size, int alignment, int or_mask) { 113 uint64_t ptr = reinterpret_cast<uint64_t>(malloc(size + 3 * alignment)); 114 if (!ptr) 115 return NULL; 116 return getAlignedMemory((uint8_t*)ptr, alignment, or_mask); 117} 118 119void initString(uint8_t *buf, size_t size) { 120 for (size_t i = 0; i < size - 1; i++) { 121 buf[i] = static_cast<char>(32 + (i % 96)); 122 } 123 buf[size-1] = '\0'; 124} 125 126static inline double computeAverage(uint64_t time_ns, size_t size, size_t copies) { 127 return ((size/1024.0) * copies) / ((double)time_ns/NS_PER_SEC); 128} 129 130static inline double computeRunningAvg(double avg, double running_avg, size_t cur_idx) { 131 return (running_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)); 132} 133 134static inline double computeRunningSquareAvg(double avg, double square_avg, size_t cur_idx) { 135 return (square_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)) * avg; 136} 137 138static inline double computeStdDev(double square_avg, double running_avg) { 139 return sqrt(square_avg - running_avg * running_avg); 140} 141 142static inline void printIter(uint64_t time_ns, const char *name, size_t size, size_t copies, double avg) { 143 printf("%s %ux%u bytes took %.06f seconds (%f MB/s)\n", 144 name, copies, size, (double)time_ns/NS_PER_SEC, avg/1024.0); 145} 146 147static inline void printSummary(uint64_t time_ns, const char *name, size_t size, size_t copies, double running_avg, double std_dev, double min, double max) { 148 printf(" %s %ux%u bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n", 149 name, copies, size, running_avg/1024.0, std_dev/1024.0, min/1024.0, 150 max/1024.0); 151} 152 153// For the cold benchmarks, a large buffer will be created which 154// contains many "size" buffers. This function will figure out the increment 155// needed between each buffer so that each one is aligned to "alignment". 156int getAlignmentIncrement(size_t size, int alignment) { 157 if (alignment == 0) { 158 alignment = DEFAULT_MALLOC_MEMORY_ALIGNMENT; 159 } 160 alignment *= 2; 161 return size + alignment - (size % alignment); 162} 163 164uint8_t *getColdBuffer(int num_buffers, size_t incr, int alignment, int or_mask) { 165 uint8_t *buffers = reinterpret_cast<uint8_t*>(malloc(num_buffers * incr + 3 * alignment)); 166 if (!buffers) { 167 return NULL; 168 } 169 return getAlignedMemory(buffers, alignment, or_mask); 170} 171 172static inline double computeColdAverage(uint64_t time_ns, size_t size, size_t copies, size_t num_buffers) { 173 return ((size/1024.0) * copies * num_buffers) / ((double)time_ns/NS_PER_SEC); 174} 175 176static void inline printColdIter(uint64_t time_ns, const char *name, size_t size, size_t copies, size_t num_buffers, double avg) { 177 printf("%s %ux%ux%u bytes took %.06f seconds (%f MB/s)\n", 178 name, copies, num_buffers, size, (double)time_ns/NS_PER_SEC, avg/1024.0); 179} 180 181static void inline printColdSummary( 182 uint64_t time_ns, const char *name, size_t size, size_t copies, size_t num_buffers, 183 double running_avg, double square_avg, double min, double max) { 184 printf(" %s %ux%ux%u bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n", 185 name, copies, num_buffers, size, running_avg/1024.0, 186 computeStdDev(running_avg, square_avg)/1024.0, min/1024.0, max/1024.0); 187} 188 189#define MAINLOOP(cmd_data, BENCH, COMPUTE_AVG, PRINT_ITER, PRINT_AVG) \ 190 uint64_t time_ns; \ 191 int iters = cmd_data.args[1]; \ 192 bool print_average = cmd_data.print_average; \ 193 bool print_each_iter = cmd_data.print_each_iter; \ 194 double min = 0.0, max = 0.0, running_avg = 0.0, square_avg = 0.0; \ 195 double avg; \ 196 for (int i = 0; iters == -1 || i < iters; i++) { \ 197 time_ns = nanoTime(); \ 198 BENCH; \ 199 time_ns = nanoTime() - time_ns; \ 200 avg = COMPUTE_AVG; \ 201 if (print_average) { \ 202 running_avg = computeRunningAvg(avg, running_avg, i); \ 203 square_avg = computeRunningSquareAvg(avg, square_avg, i); \ 204 if (min == 0.0 || avg < min) { \ 205 min = avg; \ 206 } \ 207 if (avg > max) { \ 208 max = avg; \ 209 } \ 210 } \ 211 if (print_each_iter) { \ 212 PRINT_ITER; \ 213 } \ 214 } \ 215 if (print_average) { \ 216 PRINT_AVG; \ 217 } 218 219#define MAINLOOP_DATA(name, cmd_data, size, BENCH) \ 220 size_t copies = cmd_data.data_size/size; \ 221 size_t j; \ 222 MAINLOOP(cmd_data, \ 223 for (j = 0; j < copies; j++) { \ 224 BENCH; \ 225 }, \ 226 computeAverage(time_ns, size, copies), \ 227 printIter(time_ns, name, size, copies, avg), \ 228 double std_dev = computeStdDev(square_avg, running_avg); \ 229 printSummary(time_ns, name, size, copies, running_avg, \ 230 std_dev, min, max)); 231 232#define MAINLOOP_COLD(name, cmd_data, size, num_incrs, BENCH) \ 233 size_t num_strides = num_buffers / num_incrs; \ 234 if ((num_buffers % num_incrs) != 0) { \ 235 num_strides--; \ 236 } \ 237 size_t copies = 1; \ 238 num_buffers = num_incrs * num_strides; \ 239 if (num_buffers * size < static_cast<size_t>(cmd_data.data_size)) { \ 240 copies = cmd_data.data_size / (num_buffers * size); \ 241 } \ 242 if (num_strides == 0) { \ 243 printf("%s: Chosen options lead to no copies, aborting.\n", name); \ 244 return -1; \ 245 } \ 246 size_t j, k; \ 247 MAINLOOP(cmd_data, \ 248 for (j = 0; j < copies; j++) { \ 249 for (k = 0; k < num_incrs; k++) { \ 250 BENCH; \ 251 } \ 252 }, \ 253 computeColdAverage(time_ns, size, copies, num_buffers), \ 254 printColdIter(time_ns, name, size, copies, num_buffers, avg), \ 255 printColdSummary(time_ns, name, size, copies, num_buffers, \ 256 running_avg, square_avg, min, max)); 257 258// This version of the macro creates a single buffer of the given size and 259// alignment. The variable "buf" will be a pointer to the buffer and should 260// be used by the BENCH code. 261// INIT - Any specialized code needed to initialize the data. This will only 262// be executed once. 263// BENCH - The actual code to benchmark and is timed. 264#define BENCH_ONE_BUF(name, cmd_data, INIT, BENCH) \ 265 size_t size = cmd_data.args[0]; \ 266 uint8_t *buf = allocateAlignedMemory(size, cmd_data.dst_align, cmd_data.dst_or_mask); \ 267 if (!buf) \ 268 return -1; \ 269 INIT; \ 270 MAINLOOP_DATA(name, cmd_data, size, BENCH); 271 272// This version of the macro creates two buffers of the given sizes and 273// alignments. The variables "buf1" and "buf2" will be pointers to the 274// buffers and should be used by the BENCH code. 275// INIT - Any specialized code needed to initialize the data. This will only 276// be executed once. 277// BENCH - The actual code to benchmark and is timed. 278#define BENCH_TWO_BUFS(name, cmd_data, INIT, BENCH) \ 279 size_t size = cmd_data.args[0]; \ 280 uint8_t *buf1 = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask); \ 281 if (!buf1) \ 282 return -1; \ 283 size_t total_size = size; \ 284 if (cmd_data.dst_str_size > 0) \ 285 total_size += cmd_data.dst_str_size; \ 286 uint8_t *buf2 = allocateAlignedMemory(total_size, cmd_data.dst_align, cmd_data.dst_or_mask); \ 287 if (!buf2) \ 288 return -1; \ 289 INIT; \ 290 MAINLOOP_DATA(name, cmd_data, size, BENCH); 291 292// This version of the macro attempts to benchmark code when the data 293// being manipulated is not in the cache, thus the cache is cold. It does 294// this by creating a single large buffer that is designed to be larger than 295// the largest cache in the system. The variable "buf" will be one slice 296// of the buffer that the BENCH code should use that is of the correct size 297// and alignment. In order to avoid any algorithms that prefetch past the end 298// of their "buf" and into the next sequential buffer, the code strides 299// through the buffer. Specifically, as "buf" values are iterated in BENCH 300// code, the end of "buf" is guaranteed to be at least "stride_size" away 301// from the next "buf". 302// INIT - Any specialized code needed to initialize the data. This will only 303// be executed once. 304// BENCH - The actual code to benchmark and is timed. 305#define COLD_ONE_BUF(name, cmd_data, INIT, BENCH) \ 306 size_t size = cmd_data.args[0]; \ 307 size_t incr = getAlignmentIncrement(size, cmd_data.dst_align); \ 308 size_t num_buffers = cmd_data.cold_data_size / incr; \ 309 size_t buffer_size = num_buffers * incr; \ 310 uint8_t *buffer = getColdBuffer(num_buffers, incr, cmd_data.dst_align, cmd_data.dst_or_mask); \ 311 if (!buffer) \ 312 return -1; \ 313 size_t num_incrs = cmd_data.cold_stride_size / incr + 1; \ 314 size_t stride_incr = incr * num_incrs; \ 315 uint8_t *buf; \ 316 size_t l; \ 317 INIT; \ 318 MAINLOOP_COLD(name, cmd_data, size, num_incrs, \ 319 buf = buffer + k * incr; \ 320 for (l = 0; l < num_strides; l++) { \ 321 BENCH; \ 322 buf += stride_incr; \ 323 }); 324 325// This version of the macro attempts to benchmark code when the data 326// being manipulated is not in the cache, thus the cache is cold. It does 327// this by creating two large buffers each of which is designed to be 328// larger than the largest cache in the system. Two variables "buf1" and 329// "buf2" will be the two buffers that BENCH code should use. In order 330// to avoid any algorithms that prefetch past the end of either "buf1" 331// or "buf2" and into the next sequential buffer, the code strides through 332// both buffers. Specifically, as "buf1" and "buf2" values are iterated in 333// BENCH code, the end of "buf1" and "buf2" is guaranteed to be at least 334// "stride_size" away from the next "buf1" and "buf2". 335// INIT - Any specialized code needed to initialize the data. This will only 336// be executed once. 337// BENCH - The actual code to benchmark and is timed. 338#define COLD_TWO_BUFS(name, cmd_data, INIT, BENCH) \ 339 size_t size = cmd_data.args[0]; \ 340 size_t buf1_incr = getAlignmentIncrement(size, cmd_data.src_align); \ 341 size_t total_size = size; \ 342 if (cmd_data.dst_str_size > 0) \ 343 total_size += cmd_data.dst_str_size; \ 344 size_t buf2_incr = getAlignmentIncrement(total_size, cmd_data.dst_align); \ 345 size_t max_incr = (buf1_incr > buf2_incr) ? buf1_incr : buf2_incr; \ 346 size_t num_buffers = cmd_data.cold_data_size / max_incr; \ 347 size_t buffer1_size = num_buffers * buf1_incr; \ 348 size_t buffer2_size = num_buffers * buf2_incr; \ 349 uint8_t *buffer1 = getColdBuffer(num_buffers, buf1_incr, cmd_data.src_align, cmd_data.src_or_mask); \ 350 if (!buffer1) \ 351 return -1; \ 352 uint8_t *buffer2 = getColdBuffer(num_buffers, buf2_incr, cmd_data.dst_align, cmd_data.dst_or_mask); \ 353 if (!buffer2) \ 354 return -1; \ 355 size_t min_incr = (buf1_incr < buf2_incr) ? buf1_incr : buf2_incr; \ 356 size_t num_incrs = cmd_data.cold_stride_size / min_incr + 1; \ 357 size_t buf1_stride_incr = buf1_incr * num_incrs; \ 358 size_t buf2_stride_incr = buf2_incr * num_incrs; \ 359 size_t l; \ 360 uint8_t *buf1; \ 361 uint8_t *buf2; \ 362 INIT; \ 363 MAINLOOP_COLD(name, cmd_data, size, num_incrs, \ 364 buf1 = buffer1 + k * buf1_incr; \ 365 buf2 = buffer2 + k * buf2_incr; \ 366 for (l = 0; l < num_strides; l++) { \ 367 BENCH; \ 368 buf1 += buf1_stride_incr; \ 369 buf2 += buf2_stride_incr; \ 370 }); 371 372int benchmarkSleep(const char *name, const command_data_t &cmd_data, void_func_t func) { 373 int delay = cmd_data.args[0]; 374 MAINLOOP(cmd_data, sleep(delay), 375 (double)time_ns/NS_PER_SEC, 376 printf("sleep(%d) took %.06f seconds\n", delay, avg);, 377 printf(" sleep(%d) average %.06f seconds std dev %f min %.06f seconds max %0.6f seconds\n", \ 378 delay, running_avg, computeStdDev(square_avg, running_avg), \ 379 min, max)); 380 381 return 0; 382} 383 384int benchmarkCpu(const char *name, const command_data_t &cmd_data, void_func_t func) { 385 // Use volatile so that the loop is not optimized away by the compiler. 386 volatile int cpu_foo; 387 388 MAINLOOP(cmd_data, 389 for (cpu_foo = 0; cpu_foo < 100000000; cpu_foo++), 390 (double)time_ns/NS_PER_SEC, 391 printf("cpu took %.06f seconds\n", avg), 392 printf(" cpu average %.06f seconds std dev %f min %0.6f seconds max %0.6f seconds\n", \ 393 running_avg, computeStdDev(square_avg, running_avg), min, max)); 394 395 return 0; 396} 397 398int benchmarkMemset(const char *name, const command_data_t &cmd_data, void_func_t func) { 399 memset_func_t memset_func = reinterpret_cast<memset_func_t>(func); 400 BENCH_ONE_BUF(name, cmd_data, ;, memset_func(buf, i, size)); 401 402 return 0; 403} 404 405int benchmarkMemsetCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 406 memset_func_t memset_func = reinterpret_cast<memset_func_t>(func); 407 COLD_ONE_BUF(name, cmd_data, ;, memset_func(buf, l, size)); 408 409 return 0; 410} 411 412int benchmarkMemcpy(const char *name, const command_data_t &cmd_data, void_func_t func) { 413 memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func); 414 415 BENCH_TWO_BUFS(name, cmd_data, 416 memset(buf1, 0xff, size); \ 417 memset(buf2, 0, size), 418 memcpy_func(buf2, buf1, size)); 419 420 return 0; 421} 422 423int benchmarkMemcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 424 memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func); 425 426 COLD_TWO_BUFS(name, cmd_data, 427 memset(buffer1, 0xff, buffer1_size); \ 428 memset(buffer2, 0x0, buffer2_size), 429 memcpy_func(buf2, buf1, size)); 430 431 return 0; 432} 433 434int benchmarkMemread(const char *name, const command_data_t &cmd_data, void_func_t func) { 435 int size = cmd_data.args[0]; 436 437 uint32_t *src = reinterpret_cast<uint32_t*>(malloc(size)); 438 if (!src) 439 return -1; 440 memset(src, 0xff, size); 441 442 // Use volatile so the compiler does not optimize away the reads. 443 volatile int foo; 444 size_t k; 445 MAINLOOP_DATA(name, cmd_data, size, 446 for (k = 0; k < size/sizeof(uint32_t); k++) foo = src[k]); 447 448 return 0; 449} 450 451int benchmarkStrcmp(const char *name, const command_data_t &cmd_data, void_func_t func) { 452 strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func); 453 454 int retval; 455 BENCH_TWO_BUFS(name, cmd_data, 456 initString(buf1, size); \ 457 initString(buf2, size), 458 retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \ 459 if (retval != 0) printf("%s failed, return value %d\n", name, retval)); 460 461 return 0; 462} 463 464int benchmarkStrcmpCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 465 strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func); 466 467 int retval; 468 COLD_TWO_BUFS(name, cmd_data, 469 memset(buffer1, 'a', buffer1_size); \ 470 memset(buffer2, 'a', buffer2_size); \ 471 for (size_t i =0; i < num_buffers; i++) { \ 472 buffer1[size-1+buf1_incr*i] = '\0'; \ 473 buffer2[size-1+buf2_incr*i] = '\0'; \ 474 }, 475 retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \ 476 if (retval != 0) printf("%s failed, return value %d\n", name, retval)); 477 478 return 0; 479} 480 481int benchmarkStrlen(const char *name, const command_data_t &cmd_data, void_func_t func) { 482 size_t real_size; 483 strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func); 484 BENCH_ONE_BUF(name, cmd_data, 485 initString(buf, size), 486 real_size = strlen_func(reinterpret_cast<char*>(buf)); \ 487 if (real_size + 1 != size) { \ 488 printf("%s failed, expected %u, got %u\n", name, size, real_size); \ 489 return -1; \ 490 }); 491 492 return 0; 493} 494 495int benchmarkStrlenCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 496 strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func); 497 size_t real_size; 498 COLD_ONE_BUF(name, cmd_data, 499 memset(buffer, 'a', buffer_size); \ 500 for (size_t i = 0; i < num_buffers; i++) { \ 501 buffer[size-1+incr*i] = '\0'; \ 502 }, 503 real_size = strlen_func(reinterpret_cast<char*>(buf)); \ 504 if (real_size + 1 != size) { \ 505 printf("%s failed, expected %u, got %u\n", name, size, real_size); \ 506 return -1; \ 507 }); 508 return 0; 509} 510 511int benchmarkStrcat(const char *name, const command_data_t &cmd_data, void_func_t func) { 512 str_func_t str_func = reinterpret_cast<str_func_t>(func); 513 514 int dst_str_size = cmd_data.dst_str_size; 515 if (dst_str_size <= 0) { 516 printf("%s requires --dst_str_size to be set to a non-zero value.\n", 517 name); 518 return -1; 519 } 520 BENCH_TWO_BUFS(name, cmd_data, 521 initString(buf1, size); \ 522 initString(buf2, dst_str_size), 523 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0'); 524 525 return 0; 526} 527 528int benchmarkStrcatCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 529 str_func_t str_func = reinterpret_cast<str_func_t>(func); 530 531 int dst_str_size = cmd_data.dst_str_size; 532 if (dst_str_size <= 0) { 533 printf("%s requires --dst_str_size to be set to a non-zero value.\n", 534 name); 535 return -1; 536 } 537 COLD_TWO_BUFS(name, cmd_data, 538 memset(buffer1, 'a', buffer1_size); \ 539 memset(buffer2, 'b', buffer2_size); \ 540 for (size_t i = 0; i < num_buffers; i++) { \ 541 buffer1[size-1+buf1_incr*i] = '\0'; \ 542 buffer2[dst_str_size-1+buf2_incr*i] = '\0'; \ 543 }, 544 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0'); 545 546 return 0; 547} 548 549 550int benchmarkStrcpy(const char *name, const command_data_t &cmd_data, void_func_t func) { 551 str_func_t str_func = reinterpret_cast<str_func_t>(func); 552 553 BENCH_TWO_BUFS(name, cmd_data, 554 initString(buf1, size); \ 555 memset(buf2, 0, size), 556 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1))); 557 558 return 0; 559} 560 561int benchmarkStrcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 562 str_func_t str_func = reinterpret_cast<str_func_t>(func); 563 564 COLD_TWO_BUFS(name, cmd_data, 565 memset(buffer1, 'a', buffer1_size); \ 566 for (size_t i = 0; i < num_buffers; i++) { \ 567 buffer1[size-1+buf1_incr*i] = '\0'; \ 568 } \ 569 memset(buffer2, 0, buffer2_size), 570 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1))); 571 572 return 0; 573} 574 575// Create the mapping structure. 576function_t function_table[] = { 577 { "cpu", benchmarkCpu, NULL }, 578 { "memcpy", benchmarkMemcpy, reinterpret_cast<void_func_t>(memcpy) }, 579 { "memcpy_cold", benchmarkMemcpyCold, reinterpret_cast<void_func_t>(memcpy) }, 580 { "memread", benchmarkMemread, NULL }, 581 { "memset", benchmarkMemset, reinterpret_cast<void_func_t>(memset) }, 582 { "memset_cold", benchmarkMemsetCold, reinterpret_cast<void_func_t>(memset) }, 583 { "sleep", benchmarkSleep, NULL }, 584 { "strcat", benchmarkStrcat, reinterpret_cast<void_func_t>(strcat) }, 585 { "strcat_cold", benchmarkStrcatCold, reinterpret_cast<void_func_t>(strcat) }, 586 { "strcmp", benchmarkStrcmp, reinterpret_cast<void_func_t>(strcmp) }, 587 { "strcmp_cold", benchmarkStrcmpCold, reinterpret_cast<void_func_t>(strcmp) }, 588 { "strcpy", benchmarkStrcpy, reinterpret_cast<void_func_t>(strcpy) }, 589 { "strcpy_cold", benchmarkStrcpyCold, reinterpret_cast<void_func_t>(strcpy) }, 590 { "strlen", benchmarkStrlen, reinterpret_cast<void_func_t>(strlen) }, 591 { "strlen_cold", benchmarkStrlenCold, reinterpret_cast<void_func_t>(strlen) }, 592}; 593 594void usage() { 595 printf("Usage:\n"); 596 printf(" micro_bench [--data_size DATA_BYTES] [--print_average]\n"); 597 printf(" [--no_print_each_iter] [--lock_to_cpu CORE]\n"); 598 printf(" [--src_align ALIGN] [--src_or_mask OR_MASK]\n"); 599 printf(" [--dst_align ALIGN] [--dst_or_mask OR_MASK]\n"); 600 printf(" [--dst_str_size SIZE] [--cold_data_size DATA_BYTES]\n"); 601 printf(" [--cold_stride_size SIZE]\n"); 602 printf(" --data_size DATA_BYTES\n"); 603 printf(" For the data benchmarks (memcpy/memset/memread) the approximate\n"); 604 printf(" size of data, in bytes, that will be manipulated in each iteration.\n"); 605 printf(" --print_average\n"); 606 printf(" Print the average and standard deviation of all iterations.\n"); 607 printf(" --no_print_each_iter\n"); 608 printf(" Do not print any values in each iteration.\n"); 609 printf(" --lock_to_cpu CORE\n"); 610 printf(" Lock to the specified CORE. The default is to use the last core found.\n"); 611 printf(" --dst_align ALIGN\n"); 612 printf(" If the command supports it, align the destination pointer to ALIGN.\n"); 613 printf(" The default is to use the value returned by malloc.\n"); 614 printf(" --dst_or_mask OR_MASK\n"); 615 printf(" If the command supports it, or in the OR_MASK on to the destination pointer.\n"); 616 printf(" The OR_MASK must be smaller than the dst_align value.\n"); 617 printf(" The default value is 0.\n"); 618 619 printf(" --src_align ALIGN\n"); 620 printf(" If the command supports it, align the source pointer to ALIGN. The default is to use the\n"); 621 printf(" value returned by malloc.\n"); 622 printf(" --src_or_mask OR_MASK\n"); 623 printf(" If the command supports it, or in the OR_MASK on to the source pointer.\n"); 624 printf(" The OR_MASK must be smaller than the src_align value.\n"); 625 printf(" The default value is 0.\n"); 626 printf(" --dst_str_size SIZE\n"); 627 printf(" If the command supports it, create a destination string of this length.\n"); 628 printf(" The default is to not update the destination string.\n"); 629 printf(" --cold_data_size DATA_SIZE\n"); 630 printf(" For _cold benchmarks, use this as the total amount of memory to use.\n"); 631 printf(" The default is 128MB, and the number should be larger than the cache on the chip.\n"); 632 printf(" This value is specified in bytes.\n"); 633 printf(" --cold_stride_size SIZE\n"); 634 printf(" For _cold benchmarks, use this as the minimum stride between iterations.\n"); 635 printf(" The default is 4096 bytes and the number should be larger than the amount of data\n"); 636 printf(" pulled in to the cache by each run of the benchmark.\n"); 637 printf(" ITERS\n"); 638 printf(" The number of iterations to execute each benchmark. If not\n"); 639 printf(" passed in then run forever.\n"); 640 printf(" micro_bench cpu UNUSED [ITERS]\n"); 641 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memcpy NUM_BYTES [ITERS]\n"); 642 printf(" micro_bench memread NUM_BYTES [ITERS]\n"); 643 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memset NUM_BYTES [ITERS]\n"); 644 printf(" micro_bench sleep TIME_TO_SLEEP [ITERS]\n"); 645 printf(" TIME_TO_SLEEP\n"); 646 printf(" The time in seconds to sleep.\n"); 647 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] [--dst_str_size SIZE] strcat NUM_BYTES [ITERS]\n"); 648 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask OR_MASK] strcmp NUM_BYTES [ITERS]\n"); 649 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] strcpy NUM_BYTES [ITERS]\n"); 650 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] strlen NUM_BYTES [ITERS]\n"); 651 printf("\n"); 652 printf(" In addition, memcpy/memcpy/memset/strcat/strcpy/strlen have _cold versions\n"); 653 printf(" that will execute the function on a buffer not in the cache.\n"); 654} 655 656function_t *processOptions(int argc, char **argv, command_data_t *cmd_data) { 657 function_t *command = NULL; 658 659 // Initialize the command_flags. 660 cmd_data->print_average = false; 661 cmd_data->print_each_iter = true; 662 cmd_data->dst_align = 0; 663 cmd_data->src_align = 0; 664 cmd_data->src_or_mask = 0; 665 cmd_data->dst_or_mask = 0; 666 cmd_data->num_args = 0; 667 cmd_data->cpu_to_lock = -1; 668 cmd_data->data_size = DEFAULT_DATA_SIZE; 669 cmd_data->dst_str_size = -1; 670 cmd_data->cold_data_size = DEFAULT_COLD_DATA_SIZE; 671 cmd_data->cold_stride_size = DEFAULT_COLD_STRIDE_SIZE; 672 for (int i = 0; i < MAX_ARGS; i++) { 673 cmd_data->args[i] = -1; 674 } 675 676 for (int i = 1; i < argc; i++) { 677 if (argv[i][0] == '-') { 678 int *save_value = NULL; 679 if (strcmp(argv[i], "--print_average") == 0) { 680 cmd_data->print_average = true; 681 } else if (strcmp(argv[i], "--no_print_each_iter") == 0) { 682 cmd_data->print_each_iter = false; 683 } else if (strcmp(argv[i], "--dst_align") == 0) { 684 save_value = &cmd_data->dst_align; 685 } else if (strcmp(argv[i], "--src_align") == 0) { 686 save_value = &cmd_data->src_align; 687 } else if (strcmp(argv[i], "--dst_or_mask") == 0) { 688 save_value = &cmd_data->dst_or_mask; 689 } else if (strcmp(argv[i], "--src_or_mask") == 0) { 690 save_value = &cmd_data->src_or_mask; 691 } else if (strcmp(argv[i], "--lock_to_cpu") == 0) { 692 save_value = &cmd_data->cpu_to_lock; 693 } else if (strcmp(argv[i], "--data_size") == 0) { 694 save_value = &cmd_data->data_size; 695 } else if (strcmp(argv[i], "--dst_str_size") == 0) { 696 save_value = &cmd_data->dst_str_size; 697 } else if (strcmp(argv[i], "--cold_data_size") == 0) { 698 save_value = &cmd_data->cold_data_size; 699 } else if (strcmp(argv[i], "--cold_stride_size") == 0) { 700 save_value = &cmd_data->cold_stride_size; 701 } else { 702 printf("Unknown option %s\n", argv[i]); 703 return NULL; 704 } 705 if (save_value) { 706 // Checking both characters without a strlen() call should be 707 // safe since as long as the argument exists, one character will 708 // be present (\0). And if the first character is '-', then 709 // there will always be a second character (\0 again). 710 if (i == argc - 1 || (argv[i + 1][0] == '-' && !isdigit(argv[i + 1][1]))) { 711 printf("The option %s requires one argument.\n", 712 argv[i]); 713 return NULL; 714 } 715 *save_value = (int)strtol(argv[++i], NULL, 0); 716 } 717 } else if (!command) { 718 for (size_t j = 0; j < sizeof(function_table)/sizeof(function_t); j++) { 719 if (strcmp(argv[i], function_table[j].name) == 0) { 720 command = &function_table[j]; 721 break; 722 } 723 } 724 if (!command) { 725 printf("Uknown command %s\n", argv[i]); 726 return NULL; 727 } 728 } else if (cmd_data->num_args > MAX_ARGS) { 729 printf("More than %d number arguments passed in.\n", MAX_ARGS); 730 return NULL; 731 } else { 732 cmd_data->args[cmd_data->num_args++] = atoi(argv[i]); 733 } 734 } 735 736 // Check the arguments passed in make sense. 737 if (cmd_data->num_args != 1 && cmd_data->num_args != 2) { 738 printf("Not enough arguments passed in.\n"); 739 return NULL; 740 } else if (cmd_data->dst_align < 0) { 741 printf("The --dst_align option must be greater than or equal to 0.\n"); 742 return NULL; 743 } else if (cmd_data->src_align < 0) { 744 printf("The --src_align option must be greater than or equal to 0.\n"); 745 return NULL; 746 } else if (cmd_data->data_size <= 0) { 747 printf("The --data_size option must be a positive number.\n"); 748 return NULL; 749 } else if ((cmd_data->dst_align & (cmd_data->dst_align - 1))) { 750 printf("The --dst_align option must be a power of 2.\n"); 751 return NULL; 752 } else if ((cmd_data->src_align & (cmd_data->src_align - 1))) { 753 printf("The --src_align option must be a power of 2.\n"); 754 return NULL; 755 } else if (!cmd_data->src_align && cmd_data->src_or_mask) { 756 printf("The --src_or_mask option requires that --src_align be set.\n"); 757 return NULL; 758 } else if (!cmd_data->dst_align && cmd_data->dst_or_mask) { 759 printf("The --dst_or_mask option requires that --dst_align be set.\n"); 760 return NULL; 761 } else if (cmd_data->src_or_mask > cmd_data->src_align) { 762 printf("The value of --src_or_mask cannot be larger that --src_align.\n"); 763 return NULL; 764 } else if (cmd_data->dst_or_mask > cmd_data->dst_align) { 765 printf("The value of --src_or_mask cannot be larger that --src_align.\n"); 766 return NULL; 767 } 768 769 return command; 770} 771 772bool raisePriorityAndLock(int cpu_to_lock) { 773 cpu_set_t cpuset; 774 775 if (setpriority(PRIO_PROCESS, 0, -20)) { 776 perror("Unable to raise priority of process.\n"); 777 return false; 778 } 779 780 CPU_ZERO(&cpuset); 781 if (sched_getaffinity(0, sizeof(cpuset), &cpuset) != 0) { 782 perror("sched_getaffinity failed"); 783 return false; 784 } 785 786 if (cpu_to_lock < 0) { 787 // Lock to the last active core we find. 788 for (int i = 0; i < CPU_SETSIZE; i++) { 789 if (CPU_ISSET(i, &cpuset)) { 790 cpu_to_lock = i; 791 } 792 } 793 } else if (!CPU_ISSET(cpu_to_lock, &cpuset)) { 794 printf("Cpu %d does not exist.\n", cpu_to_lock); 795 return false; 796 } 797 798 if (cpu_to_lock < 0) { 799 printf("Cannot find any valid cpu to lock.\n"); 800 return false; 801 } 802 803 CPU_ZERO(&cpuset); 804 CPU_SET(cpu_to_lock, &cpuset); 805 if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) { 806 perror("sched_setaffinity failed"); 807 return false; 808 } 809 810 return true; 811} 812 813int main(int argc, char **argv) { 814 command_data_t cmd_data; 815 816 function_t *command = processOptions(argc, argv, &cmd_data); 817 if (!command) { 818 usage(); 819 return -1; 820 } 821 822 if (!raisePriorityAndLock(cmd_data.cpu_to_lock)) { 823 return -1; 824 } 825 826 printf("%s\n", command->name); 827 return (*command->ptr)(command->name, cmd_data, command->func); 828} 829