io_u.c revision cc86c395fd9dd2002ec1edc0967b7c9453debdfb
1#include <unistd.h> 2#include <fcntl.h> 3#include <string.h> 4#include <signal.h> 5#include <time.h> 6#include <assert.h> 7 8#include "fio.h" 9#include "hash.h" 10#include "verify.h" 11#include "trim.h" 12#include "lib/rand.h" 13#include "lib/axmap.h" 14 15struct io_completion_data { 16 int nr; /* input */ 17 18 int error; /* output */ 19 uint64_t bytes_done[DDIR_RWDIR_CNT]; /* output */ 20 struct timeval time; /* output */ 21}; 22 23/* 24 * The ->io_axmap contains a map of blocks we have or have not done io 25 * to yet. Used to make sure we cover the entire range in a fair fashion. 26 */ 27static int random_map_free(struct fio_file *f, const uint64_t block) 28{ 29 return !axmap_isset(f->io_axmap, block); 30} 31 32/* 33 * Mark a given offset as used in the map. 34 */ 35static void mark_random_map(struct thread_data *td, struct io_u *io_u) 36{ 37 unsigned int min_bs = td->o.rw_min_bs; 38 struct fio_file *f = io_u->file; 39 unsigned int nr_blocks; 40 uint64_t block; 41 42 block = (io_u->offset - f->file_offset) / (uint64_t) min_bs; 43 nr_blocks = (io_u->buflen + min_bs - 1) / min_bs; 44 45 if (!(io_u->flags & IO_U_F_BUSY_OK)) 46 nr_blocks = axmap_set_nr(f->io_axmap, block, nr_blocks); 47 48 if ((nr_blocks * min_bs) < io_u->buflen) 49 io_u->buflen = nr_blocks * min_bs; 50} 51 52static uint64_t last_block(struct thread_data *td, struct fio_file *f, 53 enum fio_ddir ddir) 54{ 55 uint64_t max_blocks; 56 uint64_t max_size; 57 58 assert(ddir_rw(ddir)); 59 60 /* 61 * Hmm, should we make sure that ->io_size <= ->real_file_size? 62 */ 63 max_size = f->io_size; 64 if (max_size > f->real_file_size) 65 max_size = f->real_file_size; 66 67 if (td->o.zone_range) 68 max_size = td->o.zone_range; 69 70 max_blocks = max_size / (uint64_t) td->o.ba[ddir]; 71 if (!max_blocks) 72 return 0; 73 74 return max_blocks; 75} 76 77struct rand_off { 78 struct flist_head list; 79 uint64_t off; 80}; 81 82static int __get_next_rand_offset(struct thread_data *td, struct fio_file *f, 83 enum fio_ddir ddir, uint64_t *b) 84{ 85 uint64_t r, lastb; 86 87 lastb = last_block(td, f, ddir); 88 if (!lastb) 89 return 1; 90 91 if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE) { 92 uint64_t rmax; 93 94 rmax = td->o.use_os_rand ? OS_RAND_MAX : FRAND_MAX; 95 96 if (td->o.use_os_rand) { 97 rmax = OS_RAND_MAX; 98 r = os_random_long(&td->random_state); 99 } else { 100 rmax = FRAND_MAX; 101 r = __rand(&td->__random_state); 102 } 103 104 dprint(FD_RANDOM, "off rand %llu\n", (unsigned long long) r); 105 106 *b = (lastb - 1) * (r / ((uint64_t) rmax + 1.0)); 107 } else { 108 uint64_t off = 0; 109 110 if (lfsr_next(&f->lfsr, &off, lastb)) 111 return 1; 112 113 *b = off; 114 } 115 116 /* 117 * if we are not maintaining a random map, we are done. 118 */ 119 if (!file_randommap(td, f)) 120 goto ret; 121 122 /* 123 * calculate map offset and check if it's free 124 */ 125 if (random_map_free(f, *b)) 126 goto ret; 127 128 dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n", 129 (unsigned long long) *b); 130 131 *b = axmap_next_free(f->io_axmap, *b); 132 if (*b == (uint64_t) -1ULL) 133 return 1; 134ret: 135 return 0; 136} 137 138static int __get_next_rand_offset_zipf(struct thread_data *td, 139 struct fio_file *f, enum fio_ddir ddir, 140 uint64_t *b) 141{ 142 *b = zipf_next(&f->zipf); 143 return 0; 144} 145 146static int __get_next_rand_offset_pareto(struct thread_data *td, 147 struct fio_file *f, enum fio_ddir ddir, 148 uint64_t *b) 149{ 150 *b = pareto_next(&f->zipf); 151 return 0; 152} 153 154static int flist_cmp(void *data, struct flist_head *a, struct flist_head *b) 155{ 156 struct rand_off *r1 = flist_entry(a, struct rand_off, list); 157 struct rand_off *r2 = flist_entry(b, struct rand_off, list); 158 159 return r1->off - r2->off; 160} 161 162static int get_off_from_method(struct thread_data *td, struct fio_file *f, 163 enum fio_ddir ddir, uint64_t *b) 164{ 165 if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) 166 return __get_next_rand_offset(td, f, ddir, b); 167 else if (td->o.random_distribution == FIO_RAND_DIST_ZIPF) 168 return __get_next_rand_offset_zipf(td, f, ddir, b); 169 else if (td->o.random_distribution == FIO_RAND_DIST_PARETO) 170 return __get_next_rand_offset_pareto(td, f, ddir, b); 171 172 log_err("fio: unknown random distribution: %d\n", td->o.random_distribution); 173 return 1; 174} 175 176/* 177 * Sort the reads for a verify phase in batches of verifysort_nr, if 178 * specified. 179 */ 180static inline int should_sort_io(struct thread_data *td) 181{ 182 if (!td->o.verifysort_nr || !td->o.do_verify) 183 return 0; 184 if (!td_random(td)) 185 return 0; 186 if (td->runstate != TD_VERIFYING) 187 return 0; 188 if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE) 189 return 0; 190 191 return 1; 192} 193 194static int should_do_random(struct thread_data *td) 195{ 196 unsigned int v; 197 unsigned long r; 198 199 if (td->o.perc_rand == 100) 200 return 1; 201 202 if (td->o.use_os_rand) { 203 r = os_random_long(&td->seq_rand_state); 204 v = 1 + (int) (100.0 * (r / (OS_RAND_MAX + 1.0))); 205 } else { 206 r = __rand(&td->__seq_rand_state); 207 v = 1 + (int) (100.0 * (r / (FRAND_MAX + 1.0))); 208 } 209 210 return v <= td->o.perc_rand; 211} 212 213static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, 214 enum fio_ddir ddir, uint64_t *b) 215{ 216 struct rand_off *r; 217 int i, ret = 1; 218 219 if (!should_sort_io(td)) 220 return get_off_from_method(td, f, ddir, b); 221 222 if (!flist_empty(&td->next_rand_list)) { 223 struct rand_off *r; 224fetch: 225 r = flist_entry(td->next_rand_list.next, struct rand_off, list); 226 flist_del(&r->list); 227 *b = r->off; 228 free(r); 229 return 0; 230 } 231 232 for (i = 0; i < td->o.verifysort_nr; i++) { 233 r = malloc(sizeof(*r)); 234 235 ret = get_off_from_method(td, f, ddir, &r->off); 236 if (ret) { 237 free(r); 238 break; 239 } 240 241 flist_add(&r->list, &td->next_rand_list); 242 } 243 244 if (ret && !i) 245 return ret; 246 247 assert(!flist_empty(&td->next_rand_list)); 248 flist_sort(NULL, &td->next_rand_list, flist_cmp); 249 goto fetch; 250} 251 252static int get_next_rand_block(struct thread_data *td, struct fio_file *f, 253 enum fio_ddir ddir, uint64_t *b) 254{ 255 if (!get_next_rand_offset(td, f, ddir, b)) 256 return 0; 257 258 if (td->o.time_based) { 259 fio_file_reset(td, f); 260 if (!get_next_rand_offset(td, f, ddir, b)) 261 return 0; 262 } 263 264 dprint(FD_IO, "%s: rand offset failed, last=%llu, size=%llu\n", 265 f->file_name, (unsigned long long) f->last_pos, 266 (unsigned long long) f->real_file_size); 267 return 1; 268} 269 270static int get_next_seq_offset(struct thread_data *td, struct fio_file *f, 271 enum fio_ddir ddir, uint64_t *offset) 272{ 273 assert(ddir_rw(ddir)); 274 275 if (f->last_pos >= f->io_size + get_start_offset(td) && td->o.time_based) 276 f->last_pos = f->last_pos - f->io_size; 277 278 if (f->last_pos < f->real_file_size) { 279 uint64_t pos; 280 281 if (f->last_pos == f->file_offset && td->o.ddir_seq_add < 0) 282 f->last_pos = f->real_file_size; 283 284 pos = f->last_pos - f->file_offset; 285 if (pos) 286 pos += td->o.ddir_seq_add; 287 288 *offset = pos; 289 return 0; 290 } 291 292 return 1; 293} 294 295static int get_next_block(struct thread_data *td, struct io_u *io_u, 296 enum fio_ddir ddir, int rw_seq) 297{ 298 struct fio_file *f = io_u->file; 299 uint64_t b, offset; 300 int ret; 301 302 assert(ddir_rw(ddir)); 303 304 b = offset = -1ULL; 305 306 if (rw_seq) { 307 if (td_random(td)) { 308 if (should_do_random(td)) 309 ret = get_next_rand_block(td, f, ddir, &b); 310 else { 311 io_u->flags |= IO_U_F_BUSY_OK; 312 ret = get_next_seq_offset(td, f, ddir, &offset); 313 if (ret) 314 ret = get_next_rand_block(td, f, ddir, &b); 315 } 316 } else 317 ret = get_next_seq_offset(td, f, ddir, &offset); 318 } else { 319 io_u->flags |= IO_U_F_BUSY_OK; 320 321 if (td->o.rw_seq == RW_SEQ_SEQ) { 322 ret = get_next_seq_offset(td, f, ddir, &offset); 323 if (ret) 324 ret = get_next_rand_block(td, f, ddir, &b); 325 } else if (td->o.rw_seq == RW_SEQ_IDENT) { 326 if (f->last_start != -1ULL) 327 offset = f->last_start - f->file_offset; 328 else 329 offset = 0; 330 ret = 0; 331 } else { 332 log_err("fio: unknown rw_seq=%d\n", td->o.rw_seq); 333 ret = 1; 334 } 335 } 336 337 if (!ret) { 338 if (offset != -1ULL) 339 io_u->offset = offset; 340 else if (b != -1ULL) 341 io_u->offset = b * td->o.ba[ddir]; 342 else { 343 log_err("fio: bug in offset generation: offset=%llu, b=%llu\n", (unsigned long long) offset, (unsigned long long) b); 344 ret = 1; 345 } 346 } 347 348 return ret; 349} 350 351/* 352 * For random io, generate a random new block and see if it's used. Repeat 353 * until we find a free one. For sequential io, just return the end of 354 * the last io issued. 355 */ 356static int __get_next_offset(struct thread_data *td, struct io_u *io_u) 357{ 358 struct fio_file *f = io_u->file; 359 enum fio_ddir ddir = io_u->ddir; 360 int rw_seq_hit = 0; 361 362 assert(ddir_rw(ddir)); 363 364 if (td->o.ddir_seq_nr && !--td->ddir_seq_nr) { 365 rw_seq_hit = 1; 366 td->ddir_seq_nr = td->o.ddir_seq_nr; 367 } 368 369 if (get_next_block(td, io_u, ddir, rw_seq_hit)) 370 return 1; 371 372 if (io_u->offset >= f->io_size) { 373 dprint(FD_IO, "get_next_offset: offset %llu >= io_size %llu\n", 374 (unsigned long long) io_u->offset, 375 (unsigned long long) f->io_size); 376 return 1; 377 } 378 379 io_u->offset += f->file_offset; 380 if (io_u->offset >= f->real_file_size) { 381 dprint(FD_IO, "get_next_offset: offset %llu >= size %llu\n", 382 (unsigned long long) io_u->offset, 383 (unsigned long long) f->real_file_size); 384 return 1; 385 } 386 387 return 0; 388} 389 390static int get_next_offset(struct thread_data *td, struct io_u *io_u) 391{ 392 if (td->flags & TD_F_PROFILE_OPS) { 393 struct prof_io_ops *ops = &td->prof_io_ops; 394 395 if (ops->fill_io_u_off) 396 return ops->fill_io_u_off(td, io_u); 397 } 398 399 return __get_next_offset(td, io_u); 400} 401 402static inline int io_u_fits(struct thread_data *td, struct io_u *io_u, 403 unsigned int buflen) 404{ 405 struct fio_file *f = io_u->file; 406 407 return io_u->offset + buflen <= f->io_size + get_start_offset(td); 408} 409 410static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u) 411{ 412 const int ddir = io_u->ddir; 413 unsigned int buflen = 0; 414 unsigned int minbs, maxbs; 415 unsigned long r, rand_max; 416 417 assert(ddir_rw(ddir)); 418 419 minbs = td->o.min_bs[ddir]; 420 maxbs = td->o.max_bs[ddir]; 421 422 if (minbs == maxbs) 423 return minbs; 424 425 /* 426 * If we can't satisfy the min block size from here, then fail 427 */ 428 if (!io_u_fits(td, io_u, minbs)) 429 return 0; 430 431 if (td->o.use_os_rand) 432 rand_max = OS_RAND_MAX; 433 else 434 rand_max = FRAND_MAX; 435 436 do { 437 if (td->o.use_os_rand) 438 r = os_random_long(&td->bsrange_state); 439 else 440 r = __rand(&td->__bsrange_state); 441 442 if (!td->o.bssplit_nr[ddir]) { 443 buflen = 1 + (unsigned int) ((double) maxbs * 444 (r / (rand_max + 1.0))); 445 if (buflen < minbs) 446 buflen = minbs; 447 } else { 448 long perc = 0; 449 unsigned int i; 450 451 for (i = 0; i < td->o.bssplit_nr[ddir]; i++) { 452 struct bssplit *bsp = &td->o.bssplit[ddir][i]; 453 454 buflen = bsp->bs; 455 perc += bsp->perc; 456 if ((r <= ((rand_max / 100L) * perc)) && 457 io_u_fits(td, io_u, buflen)) 458 break; 459 } 460 } 461 462 if (!td->o.bs_unaligned && is_power_of_2(minbs)) 463 buflen = (buflen + minbs - 1) & ~(minbs - 1); 464 465 } while (!io_u_fits(td, io_u, buflen)); 466 467 return buflen; 468} 469 470static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u) 471{ 472 if (td->flags & TD_F_PROFILE_OPS) { 473 struct prof_io_ops *ops = &td->prof_io_ops; 474 475 if (ops->fill_io_u_size) 476 return ops->fill_io_u_size(td, io_u); 477 } 478 479 return __get_next_buflen(td, io_u); 480} 481 482static void set_rwmix_bytes(struct thread_data *td) 483{ 484 unsigned int diff; 485 486 /* 487 * we do time or byte based switch. this is needed because 488 * buffered writes may issue a lot quicker than they complete, 489 * whereas reads do not. 490 */ 491 diff = td->o.rwmix[td->rwmix_ddir ^ 1]; 492 td->rwmix_issues = (td->io_issues[td->rwmix_ddir] * diff) / 100; 493} 494 495static inline enum fio_ddir get_rand_ddir(struct thread_data *td) 496{ 497 unsigned int v; 498 unsigned long r; 499 500 if (td->o.use_os_rand) { 501 r = os_random_long(&td->rwmix_state); 502 v = 1 + (int) (100.0 * (r / (OS_RAND_MAX + 1.0))); 503 } else { 504 r = __rand(&td->__rwmix_state); 505 v = 1 + (int) (100.0 * (r / (FRAND_MAX + 1.0))); 506 } 507 508 if (v <= td->o.rwmix[DDIR_READ]) 509 return DDIR_READ; 510 511 return DDIR_WRITE; 512} 513 514static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) 515{ 516 enum fio_ddir odir = ddir ^ 1; 517 struct timeval t; 518 long usec; 519 520 assert(ddir_rw(ddir)); 521 522 if (td->rate_pending_usleep[ddir] <= 0) 523 return ddir; 524 525 /* 526 * We have too much pending sleep in this direction. See if we 527 * should switch. 528 */ 529 if (td_rw(td) && td->o.rwmix[odir]) { 530 /* 531 * Other direction does not have too much pending, switch 532 */ 533 if (td->rate_pending_usleep[odir] < 100000) 534 return odir; 535 536 /* 537 * Both directions have pending sleep. Sleep the minimum time 538 * and deduct from both. 539 */ 540 if (td->rate_pending_usleep[ddir] <= 541 td->rate_pending_usleep[odir]) { 542 usec = td->rate_pending_usleep[ddir]; 543 } else { 544 usec = td->rate_pending_usleep[odir]; 545 ddir = odir; 546 } 547 } else 548 usec = td->rate_pending_usleep[ddir]; 549 550 /* 551 * We are going to sleep, ensure that we flush anything pending as 552 * not to skew our latency numbers. 553 * 554 * Changed to only monitor 'in flight' requests here instead of the 555 * td->cur_depth, b/c td->cur_depth does not accurately represent 556 * io's that have been actually submitted to an async engine, 557 * and cur_depth is meaningless for sync engines. 558 */ 559 while (td->io_u_in_flight) { 560 int fio_unused ret; 561 562 ret = io_u_queued_complete(td, 1, NULL); 563 } 564 565 fio_gettime(&t, NULL); 566 usec_sleep(td, usec); 567 usec = utime_since_now(&t); 568 569 td->rate_pending_usleep[ddir] -= usec; 570 571 odir = ddir ^ 1; 572 if (td_rw(td) && __should_check_rate(td, odir)) 573 td->rate_pending_usleep[odir] -= usec; 574 575 if (ddir_trim(ddir)) 576 return ddir; 577 578 return ddir; 579} 580 581/* 582 * Return the data direction for the next io_u. If the job is a 583 * mixed read/write workload, check the rwmix cycle and switch if 584 * necessary. 585 */ 586static enum fio_ddir get_rw_ddir(struct thread_data *td) 587{ 588 enum fio_ddir ddir; 589 590 /* 591 * see if it's time to fsync 592 */ 593 if (td->o.fsync_blocks && 594 !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks) && 595 td->io_issues[DDIR_WRITE] && should_fsync(td)) 596 return DDIR_SYNC; 597 598 /* 599 * see if it's time to fdatasync 600 */ 601 if (td->o.fdatasync_blocks && 602 !(td->io_issues[DDIR_WRITE] % td->o.fdatasync_blocks) && 603 td->io_issues[DDIR_WRITE] && should_fsync(td)) 604 return DDIR_DATASYNC; 605 606 /* 607 * see if it's time to sync_file_range 608 */ 609 if (td->sync_file_range_nr && 610 !(td->io_issues[DDIR_WRITE] % td->sync_file_range_nr) && 611 td->io_issues[DDIR_WRITE] && should_fsync(td)) 612 return DDIR_SYNC_FILE_RANGE; 613 614 if (td_rw(td)) { 615 /* 616 * Check if it's time to seed a new data direction. 617 */ 618 if (td->io_issues[td->rwmix_ddir] >= td->rwmix_issues) { 619 /* 620 * Put a top limit on how many bytes we do for 621 * one data direction, to avoid overflowing the 622 * ranges too much 623 */ 624 ddir = get_rand_ddir(td); 625 626 if (ddir != td->rwmix_ddir) 627 set_rwmix_bytes(td); 628 629 td->rwmix_ddir = ddir; 630 } 631 ddir = td->rwmix_ddir; 632 } else if (td_read(td)) 633 ddir = DDIR_READ; 634 else if (td_write(td)) 635 ddir = DDIR_WRITE; 636 else 637 ddir = DDIR_TRIM; 638 639 td->rwmix_ddir = rate_ddir(td, ddir); 640 return td->rwmix_ddir; 641} 642 643static void set_rw_ddir(struct thread_data *td, struct io_u *io_u) 644{ 645 io_u->ddir = io_u->acct_ddir = get_rw_ddir(td); 646 647 if (io_u->ddir == DDIR_WRITE && (td->io_ops->flags & FIO_BARRIER) && 648 td->o.barrier_blocks && 649 !(td->io_issues[DDIR_WRITE] % td->o.barrier_blocks) && 650 td->io_issues[DDIR_WRITE]) 651 io_u->flags |= IO_U_F_BARRIER; 652} 653 654void put_file_log(struct thread_data *td, struct fio_file *f) 655{ 656 int ret = put_file(td, f); 657 658 if (ret) 659 td_verror(td, ret, "file close"); 660} 661 662void put_io_u(struct thread_data *td, struct io_u *io_u) 663{ 664 td_io_u_lock(td); 665 666 if (io_u->file && !(io_u->flags & IO_U_F_FREE_DEF)) 667 put_file_log(td, io_u->file); 668 io_u->file = NULL; 669 io_u->flags &= ~IO_U_F_FREE_DEF; 670 io_u->flags |= IO_U_F_FREE; 671 672 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) 673 td->cur_depth--; 674 flist_del_init(&io_u->list); 675 flist_add(&io_u->list, &td->io_u_freelist); 676 td_io_u_unlock(td); 677 td_io_u_free_notify(td); 678} 679 680void clear_io_u(struct thread_data *td, struct io_u *io_u) 681{ 682 io_u->flags &= ~IO_U_F_FLIGHT; 683 put_io_u(td, io_u); 684} 685 686void requeue_io_u(struct thread_data *td, struct io_u **io_u) 687{ 688 struct io_u *__io_u = *io_u; 689 enum fio_ddir ddir = acct_ddir(__io_u); 690 691 dprint(FD_IO, "requeue %p\n", __io_u); 692 693 td_io_u_lock(td); 694 695 __io_u->flags |= IO_U_F_FREE; 696 if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(ddir)) 697 td->io_issues[ddir]--; 698 699 __io_u->flags &= ~IO_U_F_FLIGHT; 700 if (__io_u->flags & IO_U_F_IN_CUR_DEPTH) 701 td->cur_depth--; 702 flist_del(&__io_u->list); 703 flist_add_tail(&__io_u->list, &td->io_u_requeues); 704 td_io_u_unlock(td); 705 *io_u = NULL; 706} 707 708static int fill_io_u(struct thread_data *td, struct io_u *io_u) 709{ 710 if (td->io_ops->flags & FIO_NOIO) 711 goto out; 712 713 set_rw_ddir(td, io_u); 714 715 /* 716 * fsync() or fdatasync() or trim etc, we are done 717 */ 718 if (!ddir_rw(io_u->ddir)) 719 goto out; 720 721 /* 722 * See if it's time to switch to a new zone 723 */ 724 if (td->zone_bytes >= td->o.zone_size && td->o.zone_skip) { 725 td->zone_bytes = 0; 726 io_u->file->file_offset += td->o.zone_range + td->o.zone_skip; 727 io_u->file->last_pos = io_u->file->file_offset; 728 td->io_skip_bytes += td->o.zone_skip; 729 } 730 731 /* 732 * No log, let the seq/rand engine retrieve the next buflen and 733 * position. 734 */ 735 if (get_next_offset(td, io_u)) { 736 dprint(FD_IO, "io_u %p, failed getting offset\n", io_u); 737 return 1; 738 } 739 740 io_u->buflen = get_next_buflen(td, io_u); 741 if (!io_u->buflen) { 742 dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u); 743 return 1; 744 } 745 746 if (io_u->offset + io_u->buflen > io_u->file->real_file_size) { 747 dprint(FD_IO, "io_u %p, offset too large\n", io_u); 748 dprint(FD_IO, " off=%llu/%lu > %llu\n", 749 (unsigned long long) io_u->offset, io_u->buflen, 750 (unsigned long long) io_u->file->real_file_size); 751 return 1; 752 } 753 754 /* 755 * mark entry before potentially trimming io_u 756 */ 757 if (td_random(td) && file_randommap(td, io_u->file)) 758 mark_random_map(td, io_u); 759 760out: 761 dprint_io_u(io_u, "fill_io_u"); 762 td->zone_bytes += io_u->buflen; 763 return 0; 764} 765 766static void __io_u_mark_map(unsigned int *map, unsigned int nr) 767{ 768 int idx = 0; 769 770 switch (nr) { 771 default: 772 idx = 6; 773 break; 774 case 33 ... 64: 775 idx = 5; 776 break; 777 case 17 ... 32: 778 idx = 4; 779 break; 780 case 9 ... 16: 781 idx = 3; 782 break; 783 case 5 ... 8: 784 idx = 2; 785 break; 786 case 1 ... 4: 787 idx = 1; 788 case 0: 789 break; 790 } 791 792 map[idx]++; 793} 794 795void io_u_mark_submit(struct thread_data *td, unsigned int nr) 796{ 797 __io_u_mark_map(td->ts.io_u_submit, nr); 798 td->ts.total_submit++; 799} 800 801void io_u_mark_complete(struct thread_data *td, unsigned int nr) 802{ 803 __io_u_mark_map(td->ts.io_u_complete, nr); 804 td->ts.total_complete++; 805} 806 807void io_u_mark_depth(struct thread_data *td, unsigned int nr) 808{ 809 int idx = 0; 810 811 switch (td->cur_depth) { 812 default: 813 idx = 6; 814 break; 815 case 32 ... 63: 816 idx = 5; 817 break; 818 case 16 ... 31: 819 idx = 4; 820 break; 821 case 8 ... 15: 822 idx = 3; 823 break; 824 case 4 ... 7: 825 idx = 2; 826 break; 827 case 2 ... 3: 828 idx = 1; 829 case 1: 830 break; 831 } 832 833 td->ts.io_u_map[idx] += nr; 834} 835 836static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec) 837{ 838 int idx = 0; 839 840 assert(usec < 1000); 841 842 switch (usec) { 843 case 750 ... 999: 844 idx = 9; 845 break; 846 case 500 ... 749: 847 idx = 8; 848 break; 849 case 250 ... 499: 850 idx = 7; 851 break; 852 case 100 ... 249: 853 idx = 6; 854 break; 855 case 50 ... 99: 856 idx = 5; 857 break; 858 case 20 ... 49: 859 idx = 4; 860 break; 861 case 10 ... 19: 862 idx = 3; 863 break; 864 case 4 ... 9: 865 idx = 2; 866 break; 867 case 2 ... 3: 868 idx = 1; 869 case 0 ... 1: 870 break; 871 } 872 873 assert(idx < FIO_IO_U_LAT_U_NR); 874 td->ts.io_u_lat_u[idx]++; 875} 876 877static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec) 878{ 879 int idx = 0; 880 881 switch (msec) { 882 default: 883 idx = 11; 884 break; 885 case 1000 ... 1999: 886 idx = 10; 887 break; 888 case 750 ... 999: 889 idx = 9; 890 break; 891 case 500 ... 749: 892 idx = 8; 893 break; 894 case 250 ... 499: 895 idx = 7; 896 break; 897 case 100 ... 249: 898 idx = 6; 899 break; 900 case 50 ... 99: 901 idx = 5; 902 break; 903 case 20 ... 49: 904 idx = 4; 905 break; 906 case 10 ... 19: 907 idx = 3; 908 break; 909 case 4 ... 9: 910 idx = 2; 911 break; 912 case 2 ... 3: 913 idx = 1; 914 case 0 ... 1: 915 break; 916 } 917 918 assert(idx < FIO_IO_U_LAT_M_NR); 919 td->ts.io_u_lat_m[idx]++; 920} 921 922static void io_u_mark_latency(struct thread_data *td, unsigned long usec) 923{ 924 if (usec < 1000) 925 io_u_mark_lat_usec(td, usec); 926 else 927 io_u_mark_lat_msec(td, usec / 1000); 928} 929 930/* 931 * Get next file to service by choosing one at random 932 */ 933static struct fio_file *get_next_file_rand(struct thread_data *td, 934 enum fio_file_flags goodf, 935 enum fio_file_flags badf) 936{ 937 struct fio_file *f; 938 int fno; 939 940 do { 941 int opened = 0; 942 unsigned long r; 943 944 if (td->o.use_os_rand) { 945 r = os_random_long(&td->next_file_state); 946 fno = (unsigned int) ((double) td->o.nr_files 947 * (r / (OS_RAND_MAX + 1.0))); 948 } else { 949 r = __rand(&td->__next_file_state); 950 fno = (unsigned int) ((double) td->o.nr_files 951 * (r / (FRAND_MAX + 1.0))); 952 } 953 954 f = td->files[fno]; 955 if (fio_file_done(f)) 956 continue; 957 958 if (!fio_file_open(f)) { 959 int err; 960 961 err = td_io_open_file(td, f); 962 if (err) 963 continue; 964 opened = 1; 965 } 966 967 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) { 968 dprint(FD_FILE, "get_next_file_rand: %p\n", f); 969 return f; 970 } 971 if (opened) 972 td_io_close_file(td, f); 973 } while (1); 974} 975 976/* 977 * Get next file to service by doing round robin between all available ones 978 */ 979static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf, 980 int badf) 981{ 982 unsigned int old_next_file = td->next_file; 983 struct fio_file *f; 984 985 do { 986 int opened = 0; 987 988 f = td->files[td->next_file]; 989 990 td->next_file++; 991 if (td->next_file >= td->o.nr_files) 992 td->next_file = 0; 993 994 dprint(FD_FILE, "trying file %s %x\n", f->file_name, f->flags); 995 if (fio_file_done(f)) { 996 f = NULL; 997 continue; 998 } 999 1000 if (!fio_file_open(f)) { 1001 int err; 1002 1003 err = td_io_open_file(td, f); 1004 if (err) { 1005 dprint(FD_FILE, "error %d on open of %s\n", 1006 err, f->file_name); 1007 f = NULL; 1008 continue; 1009 } 1010 opened = 1; 1011 } 1012 1013 dprint(FD_FILE, "goodf=%x, badf=%x, ff=%x\n", goodf, badf, 1014 f->flags); 1015 if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) 1016 break; 1017 1018 if (opened) 1019 td_io_close_file(td, f); 1020 1021 f = NULL; 1022 } while (td->next_file != old_next_file); 1023 1024 dprint(FD_FILE, "get_next_file_rr: %p\n", f); 1025 return f; 1026} 1027 1028static struct fio_file *__get_next_file(struct thread_data *td) 1029{ 1030 struct fio_file *f; 1031 1032 assert(td->o.nr_files <= td->files_index); 1033 1034 if (td->nr_done_files >= td->o.nr_files) { 1035 dprint(FD_FILE, "get_next_file: nr_open=%d, nr_done=%d," 1036 " nr_files=%d\n", td->nr_open_files, 1037 td->nr_done_files, 1038 td->o.nr_files); 1039 return NULL; 1040 } 1041 1042 f = td->file_service_file; 1043 if (f && fio_file_open(f) && !fio_file_closing(f)) { 1044 if (td->o.file_service_type == FIO_FSERVICE_SEQ) 1045 goto out; 1046 if (td->file_service_left--) 1047 goto out; 1048 } 1049 1050 if (td->o.file_service_type == FIO_FSERVICE_RR || 1051 td->o.file_service_type == FIO_FSERVICE_SEQ) 1052 f = get_next_file_rr(td, FIO_FILE_open, FIO_FILE_closing); 1053 else 1054 f = get_next_file_rand(td, FIO_FILE_open, FIO_FILE_closing); 1055 1056 td->file_service_file = f; 1057 td->file_service_left = td->file_service_nr - 1; 1058out: 1059 dprint(FD_FILE, "get_next_file: %p [%s]\n", f, f->file_name); 1060 return f; 1061} 1062 1063static struct fio_file *get_next_file(struct thread_data *td) 1064{ 1065 if (!(td->flags & TD_F_PROFILE_OPS)) { 1066 struct prof_io_ops *ops = &td->prof_io_ops; 1067 1068 if (ops->get_next_file) 1069 return ops->get_next_file(td); 1070 } 1071 1072 return __get_next_file(td); 1073} 1074 1075static int set_io_u_file(struct thread_data *td, struct io_u *io_u) 1076{ 1077 struct fio_file *f; 1078 1079 do { 1080 f = get_next_file(td); 1081 if (!f) 1082 return 1; 1083 1084 io_u->file = f; 1085 get_file(f); 1086 1087 if (!fill_io_u(td, io_u)) 1088 break; 1089 1090 put_file_log(td, f); 1091 td_io_close_file(td, f); 1092 io_u->file = NULL; 1093 fio_file_set_done(f); 1094 td->nr_done_files++; 1095 dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name, 1096 td->nr_done_files, td->o.nr_files); 1097 } while (1); 1098 1099 return 0; 1100} 1101 1102 1103struct io_u *__get_io_u(struct thread_data *td) 1104{ 1105 struct io_u *io_u = NULL; 1106 1107 td_io_u_lock(td); 1108 1109again: 1110 if (!flist_empty(&td->io_u_requeues)) 1111 io_u = flist_entry(td->io_u_requeues.next, struct io_u, list); 1112 else if (!queue_full(td)) { 1113 io_u = flist_entry(td->io_u_freelist.next, struct io_u, list); 1114 1115 io_u->buflen = 0; 1116 io_u->resid = 0; 1117 io_u->file = NULL; 1118 io_u->end_io = NULL; 1119 } 1120 1121 if (io_u) { 1122 assert(io_u->flags & IO_U_F_FREE); 1123 io_u->flags &= ~(IO_U_F_FREE | IO_U_F_FREE_DEF); 1124 io_u->flags &= ~(IO_U_F_TRIMMED | IO_U_F_BARRIER); 1125 io_u->flags &= ~IO_U_F_VER_LIST; 1126 1127 io_u->error = 0; 1128 io_u->acct_ddir = -1; 1129 flist_del(&io_u->list); 1130 flist_add_tail(&io_u->list, &td->io_u_busylist); 1131 td->cur_depth++; 1132 io_u->flags |= IO_U_F_IN_CUR_DEPTH; 1133 } else if (td->o.verify_async) { 1134 /* 1135 * We ran out, wait for async verify threads to finish and 1136 * return one 1137 */ 1138 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1139 goto again; 1140 } 1141 1142 td_io_u_unlock(td); 1143 return io_u; 1144} 1145 1146static int check_get_trim(struct thread_data *td, struct io_u *io_u) 1147{ 1148 if (!(td->flags & TD_F_TRIM_BACKLOG)) 1149 return 0; 1150 1151 if (td->trim_entries) { 1152 int get_trim = 0; 1153 1154 if (td->trim_batch) { 1155 td->trim_batch--; 1156 get_trim = 1; 1157 } else if (!(td->io_hist_len % td->o.trim_backlog) && 1158 td->last_ddir != DDIR_READ) { 1159 td->trim_batch = td->o.trim_batch; 1160 if (!td->trim_batch) 1161 td->trim_batch = td->o.trim_backlog; 1162 get_trim = 1; 1163 } 1164 1165 if (get_trim && !get_next_trim(td, io_u)) 1166 return 1; 1167 } 1168 1169 return 0; 1170} 1171 1172static int check_get_verify(struct thread_data *td, struct io_u *io_u) 1173{ 1174 if (!(td->flags & TD_F_VER_BACKLOG)) 1175 return 0; 1176 1177 if (td->io_hist_len) { 1178 int get_verify = 0; 1179 1180 if (td->verify_batch) 1181 get_verify = 1; 1182 else if (!(td->io_hist_len % td->o.verify_backlog) && 1183 td->last_ddir != DDIR_READ) { 1184 td->verify_batch = td->o.verify_batch; 1185 if (!td->verify_batch) 1186 td->verify_batch = td->o.verify_backlog; 1187 get_verify = 1; 1188 } 1189 1190 if (get_verify && !get_next_verify(td, io_u)) { 1191 td->verify_batch--; 1192 return 1; 1193 } 1194 } 1195 1196 return 0; 1197} 1198 1199/* 1200 * Fill offset and start time into the buffer content, to prevent too 1201 * easy compressible data for simple de-dupe attempts. Do this for every 1202 * 512b block in the range, since that should be the smallest block size 1203 * we can expect from a device. 1204 */ 1205static void small_content_scramble(struct io_u *io_u) 1206{ 1207 unsigned int i, nr_blocks = io_u->buflen / 512; 1208 uint64_t boffset; 1209 unsigned int offset; 1210 void *p, *end; 1211 1212 if (!nr_blocks) 1213 return; 1214 1215 p = io_u->xfer_buf; 1216 boffset = io_u->offset; 1217 io_u->buf_filled_len = 0; 1218 1219 for (i = 0; i < nr_blocks; i++) { 1220 /* 1221 * Fill the byte offset into a "random" start offset of 1222 * the buffer, given by the product of the usec time 1223 * and the actual offset. 1224 */ 1225 offset = (io_u->start_time.tv_usec ^ boffset) & 511; 1226 offset &= ~(sizeof(uint64_t) - 1); 1227 if (offset >= 512 - sizeof(uint64_t)) 1228 offset -= sizeof(uint64_t); 1229 memcpy(p + offset, &boffset, sizeof(boffset)); 1230 1231 end = p + 512 - sizeof(io_u->start_time); 1232 memcpy(end, &io_u->start_time, sizeof(io_u->start_time)); 1233 p += 512; 1234 boffset += 512; 1235 } 1236} 1237 1238/* 1239 * Return an io_u to be processed. Gets a buflen and offset, sets direction, 1240 * etc. The returned io_u is fully ready to be prepped and submitted. 1241 */ 1242struct io_u *get_io_u(struct thread_data *td) 1243{ 1244 struct fio_file *f; 1245 struct io_u *io_u; 1246 int do_scramble = 0; 1247 1248 io_u = __get_io_u(td); 1249 if (!io_u) { 1250 dprint(FD_IO, "__get_io_u failed\n"); 1251 return NULL; 1252 } 1253 1254 if (check_get_verify(td, io_u)) 1255 goto out; 1256 if (check_get_trim(td, io_u)) 1257 goto out; 1258 1259 /* 1260 * from a requeue, io_u already setup 1261 */ 1262 if (io_u->file) 1263 goto out; 1264 1265 /* 1266 * If using an iolog, grab next piece if any available. 1267 */ 1268 if (td->flags & TD_F_READ_IOLOG) { 1269 if (read_iolog_get(td, io_u)) 1270 goto err_put; 1271 } else if (set_io_u_file(td, io_u)) { 1272 dprint(FD_IO, "io_u %p, setting file failed\n", io_u); 1273 goto err_put; 1274 } 1275 1276 f = io_u->file; 1277 assert(fio_file_open(f)); 1278 1279 if (ddir_rw(io_u->ddir)) { 1280 if (!io_u->buflen && !(td->io_ops->flags & FIO_NOIO)) { 1281 dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u); 1282 goto err_put; 1283 } 1284 1285 f->last_start = io_u->offset; 1286 f->last_pos = io_u->offset + io_u->buflen; 1287 1288 if (io_u->ddir == DDIR_WRITE) { 1289 if (td->flags & TD_F_REFILL_BUFFERS) { 1290 io_u_fill_buffer(td, io_u, 1291 io_u->xfer_buflen, io_u->xfer_buflen); 1292 } else if (td->flags & TD_F_SCRAMBLE_BUFFERS) 1293 do_scramble = 1; 1294 if (td->flags & TD_F_VER_NONE) { 1295 populate_verify_io_u(td, io_u); 1296 do_scramble = 0; 1297 } 1298 } else if (io_u->ddir == DDIR_READ) { 1299 /* 1300 * Reset the buf_filled parameters so next time if the 1301 * buffer is used for writes it is refilled. 1302 */ 1303 io_u->buf_filled_len = 0; 1304 } 1305 } 1306 1307 /* 1308 * Set io data pointers. 1309 */ 1310 io_u->xfer_buf = io_u->buf; 1311 io_u->xfer_buflen = io_u->buflen; 1312 1313out: 1314 assert(io_u->file); 1315 if (!td_io_prep(td, io_u)) { 1316 if (!td->o.disable_slat) 1317 fio_gettime(&io_u->start_time, NULL); 1318 if (do_scramble) 1319 small_content_scramble(io_u); 1320 return io_u; 1321 } 1322err_put: 1323 dprint(FD_IO, "get_io_u failed\n"); 1324 put_io_u(td, io_u); 1325 return NULL; 1326} 1327 1328void io_u_log_error(struct thread_data *td, struct io_u *io_u) 1329{ 1330 enum error_type_bit eb = td_error_type(io_u->ddir, io_u->error); 1331 const char *msg[] = { "read", "write", "sync", "datasync", 1332 "sync_file_range", "wait", "trim" }; 1333 1334 if (td_non_fatal_error(td, eb, io_u->error) && !td->o.error_dump) 1335 return; 1336 1337 log_err("fio: io_u error"); 1338 1339 if (io_u->file) 1340 log_err(" on file %s", io_u->file->file_name); 1341 1342 log_err(": %s\n", strerror(io_u->error)); 1343 1344 log_err(" %s offset=%llu, buflen=%lu\n", msg[io_u->ddir], 1345 io_u->offset, io_u->xfer_buflen); 1346 1347 if (!td->error) 1348 td_verror(td, io_u->error, "io_u error"); 1349} 1350 1351static void account_io_completion(struct thread_data *td, struct io_u *io_u, 1352 struct io_completion_data *icd, 1353 const enum fio_ddir idx, unsigned int bytes) 1354{ 1355 unsigned long lusec = 0; 1356 1357 if (!td->o.disable_clat || !td->o.disable_bw) 1358 lusec = utime_since(&io_u->issue_time, &icd->time); 1359 1360 if (!td->o.disable_lat) { 1361 unsigned long tusec; 1362 1363 tusec = utime_since(&io_u->start_time, &icd->time); 1364 add_lat_sample(td, idx, tusec, bytes); 1365 1366 if (td->o.max_latency && tusec > td->o.max_latency) { 1367 if (!td->error) 1368 log_err("fio: latency of %lu usec exceeds specified max (%u usec)\n", tusec, td->o.max_latency); 1369 td_verror(td, ETIMEDOUT, "max latency exceeded"); 1370 icd->error = ETIMEDOUT; 1371 } 1372 } 1373 1374 if (!td->o.disable_clat) { 1375 add_clat_sample(td, idx, lusec, bytes); 1376 io_u_mark_latency(td, lusec); 1377 } 1378 1379 if (!td->o.disable_bw) 1380 add_bw_sample(td, idx, bytes, &icd->time); 1381 1382 add_iops_sample(td, idx, &icd->time); 1383} 1384 1385static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir) 1386{ 1387 uint64_t secs, remainder, bps, bytes; 1388 1389 bytes = td->this_io_bytes[ddir]; 1390 bps = td->rate_bps[ddir]; 1391 secs = bytes / bps; 1392 remainder = bytes % bps; 1393 return remainder * 1000000 / bps + secs * 1000000; 1394} 1395 1396static void io_completed(struct thread_data *td, struct io_u *io_u, 1397 struct io_completion_data *icd) 1398{ 1399 struct fio_file *f; 1400 1401 dprint_io_u(io_u, "io complete"); 1402 1403 td_io_u_lock(td); 1404 assert(io_u->flags & IO_U_F_FLIGHT); 1405 io_u->flags &= ~(IO_U_F_FLIGHT | IO_U_F_BUSY_OK); 1406 td_io_u_unlock(td); 1407 1408 if (ddir_sync(io_u->ddir)) { 1409 td->last_was_sync = 1; 1410 f = io_u->file; 1411 if (f) { 1412 f->first_write = -1ULL; 1413 f->last_write = -1ULL; 1414 } 1415 return; 1416 } 1417 1418 td->last_was_sync = 0; 1419 td->last_ddir = io_u->ddir; 1420 1421 if (!io_u->error && ddir_rw(io_u->ddir)) { 1422 unsigned int bytes = io_u->buflen - io_u->resid; 1423 const enum fio_ddir idx = io_u->ddir; 1424 const enum fio_ddir odx = io_u->ddir ^ 1; 1425 int ret; 1426 1427 td->io_blocks[idx]++; 1428 td->this_io_blocks[idx]++; 1429 td->io_bytes[idx] += bytes; 1430 1431 if (!(io_u->flags & IO_U_F_VER_LIST)) 1432 td->this_io_bytes[idx] += bytes; 1433 1434 if (idx == DDIR_WRITE) { 1435 f = io_u->file; 1436 if (f) { 1437 if (f->first_write == -1ULL || 1438 io_u->offset < f->first_write) 1439 f->first_write = io_u->offset; 1440 if (f->last_write == -1ULL || 1441 ((io_u->offset + bytes) > f->last_write)) 1442 f->last_write = io_u->offset + bytes; 1443 } 1444 } 1445 1446 if (ramp_time_over(td) && (td->runstate == TD_RUNNING || 1447 td->runstate == TD_VERIFYING)) { 1448 account_io_completion(td, io_u, icd, idx, bytes); 1449 1450 if (__should_check_rate(td, idx)) { 1451 td->rate_pending_usleep[idx] = 1452 (usec_for_io(td, idx) - 1453 utime_since_now(&td->start)); 1454 } 1455 if (idx != DDIR_TRIM && __should_check_rate(td, odx)) 1456 td->rate_pending_usleep[odx] = 1457 (usec_for_io(td, odx) - 1458 utime_since_now(&td->start)); 1459 } 1460 1461 if (td_write(td) && idx == DDIR_WRITE && 1462 td->o.do_verify && 1463 td->o.verify != VERIFY_NONE && 1464 !td->o.experimental_verify) 1465 log_io_piece(td, io_u); 1466 1467 icd->bytes_done[idx] += bytes; 1468 1469 if (io_u->end_io) { 1470 ret = io_u->end_io(td, io_u); 1471 if (ret && !icd->error) 1472 icd->error = ret; 1473 } 1474 } else if (io_u->error) { 1475 icd->error = io_u->error; 1476 io_u_log_error(td, io_u); 1477 } 1478 if (icd->error) { 1479 enum error_type_bit eb = td_error_type(io_u->ddir, icd->error); 1480 if (!td_non_fatal_error(td, eb, icd->error)) 1481 return; 1482 /* 1483 * If there is a non_fatal error, then add to the error count 1484 * and clear all the errors. 1485 */ 1486 update_error_count(td, icd->error); 1487 td_clear_error(td); 1488 icd->error = 0; 1489 io_u->error = 0; 1490 } 1491} 1492 1493static void init_icd(struct thread_data *td, struct io_completion_data *icd, 1494 int nr) 1495{ 1496 int ddir; 1497 if (!td->o.disable_clat || !td->o.disable_bw) 1498 fio_gettime(&icd->time, NULL); 1499 1500 icd->nr = nr; 1501 1502 icd->error = 0; 1503 for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) 1504 icd->bytes_done[ddir] = 0; 1505} 1506 1507static void ios_completed(struct thread_data *td, 1508 struct io_completion_data *icd) 1509{ 1510 struct io_u *io_u; 1511 int i; 1512 1513 for (i = 0; i < icd->nr; i++) { 1514 io_u = td->io_ops->event(td, i); 1515 1516 io_completed(td, io_u, icd); 1517 1518 if (!(io_u->flags & IO_U_F_FREE_DEF)) 1519 put_io_u(td, io_u); 1520 } 1521} 1522 1523/* 1524 * Complete a single io_u for the sync engines. 1525 */ 1526int io_u_sync_complete(struct thread_data *td, struct io_u *io_u, 1527 uint64_t *bytes) 1528{ 1529 struct io_completion_data icd; 1530 1531 init_icd(td, &icd, 1); 1532 io_completed(td, io_u, &icd); 1533 1534 if (!(io_u->flags & IO_U_F_FREE_DEF)) 1535 put_io_u(td, io_u); 1536 1537 if (icd.error) { 1538 td_verror(td, icd.error, "io_u_sync_complete"); 1539 return -1; 1540 } 1541 1542 if (bytes) { 1543 int ddir; 1544 1545 for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) 1546 bytes[ddir] += icd.bytes_done[ddir]; 1547 } 1548 1549 return 0; 1550} 1551 1552/* 1553 * Called to complete min_events number of io for the async engines. 1554 */ 1555int io_u_queued_complete(struct thread_data *td, int min_evts, 1556 uint64_t *bytes) 1557{ 1558 struct io_completion_data icd; 1559 struct timespec *tvp = NULL; 1560 int ret; 1561 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, }; 1562 1563 dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_evts); 1564 1565 if (!min_evts) 1566 tvp = &ts; 1567 1568 ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete, tvp); 1569 if (ret < 0) { 1570 td_verror(td, -ret, "td_io_getevents"); 1571 return ret; 1572 } else if (!ret) 1573 return ret; 1574 1575 init_icd(td, &icd, ret); 1576 ios_completed(td, &icd); 1577 if (icd.error) { 1578 td_verror(td, icd.error, "io_u_queued_complete"); 1579 return -1; 1580 } 1581 1582 if (bytes) { 1583 int ddir; 1584 1585 for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) 1586 bytes[ddir] += icd.bytes_done[ddir]; 1587 } 1588 1589 return 0; 1590} 1591 1592/* 1593 * Call when io_u is really queued, to update the submission latency. 1594 */ 1595void io_u_queued(struct thread_data *td, struct io_u *io_u) 1596{ 1597 if (!td->o.disable_slat) { 1598 unsigned long slat_time; 1599 1600 slat_time = utime_since(&io_u->start_time, &io_u->issue_time); 1601 add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen); 1602 } 1603} 1604 1605void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, 1606 unsigned int max_bs) 1607{ 1608 if (!td->o.zero_buffers) { 1609 unsigned int perc = td->o.compress_percentage; 1610 1611 if (perc) { 1612 unsigned int seg = min_write; 1613 1614 seg = min(min_write, td->o.compress_chunk); 1615 if (!seg) 1616 seg = min_write; 1617 1618 fill_random_buf_percentage(&td->buf_state, buf, 1619 perc, seg, max_bs); 1620 } else 1621 fill_random_buf(&td->buf_state, buf, max_bs); 1622 } else 1623 memset(buf, 0, max_bs); 1624} 1625 1626/* 1627 * "randomly" fill the buffer contents 1628 */ 1629void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u, 1630 unsigned int min_write, unsigned int max_bs) 1631{ 1632 io_u->buf_filled_len = 0; 1633 fill_io_buffer(td, io_u->buf, min_write, max_bs); 1634} 1635