filesetup.c revision dfe11fd1e8249a52c42f34c9e9e6b910ab6b05c5
1#include <unistd.h> 2#include <fcntl.h> 3#include <string.h> 4#include <assert.h> 5#include <dirent.h> 6#include <libgen.h> 7#include <sys/stat.h> 8#include <sys/mman.h> 9#include <sys/types.h> 10 11#include "fio.h" 12#include "smalloc.h" 13#include "filehash.h" 14#include "options.h" 15#include "os/os.h" 16#include "hash.h" 17#include "lib/axmap.h" 18 19#ifdef CONFIG_LINUX_FALLOCATE 20#include <linux/falloc.h> 21#endif 22 23static int root_warn; 24 25static FLIST_HEAD(filename_list); 26 27static inline void clear_error(struct thread_data *td) 28{ 29 td->error = 0; 30 td->verror[0] = '\0'; 31} 32 33/* 34 * Leaves f->fd open on success, caller must close 35 */ 36static int extend_file(struct thread_data *td, struct fio_file *f) 37{ 38 int r, new_layout = 0, unlink_file = 0, flags; 39 unsigned long long left; 40 unsigned int bs; 41 char *b; 42 43 if (read_only) { 44 log_err("fio: refusing extend of file due to read-only\n"); 45 return 0; 46 } 47 48 /* 49 * check if we need to lay the file out complete again. fio 50 * does that for operations involving reads, or for writes 51 * where overwrite is set 52 */ 53 if (td_read(td) || 54 (td_write(td) && td->o.overwrite && !td->o.file_append) || 55 (td_write(td) && td->io_ops->flags & FIO_NOEXTEND)) 56 new_layout = 1; 57 if (td_write(td) && !td->o.overwrite && !td->o.file_append) 58 unlink_file = 1; 59 60 if (unlink_file || new_layout) { 61 dprint(FD_FILE, "layout unlink %s\n", f->file_name); 62 if ((unlink(f->file_name) < 0) && (errno != ENOENT)) { 63 td_verror(td, errno, "unlink"); 64 return 1; 65 } 66 } 67 68 flags = O_WRONLY | O_CREAT; 69 if (new_layout) 70 flags |= O_TRUNC; 71 72 dprint(FD_FILE, "open file %s, flags %x\n", f->file_name, flags); 73 f->fd = open(f->file_name, flags, 0644); 74 if (f->fd < 0) { 75 td_verror(td, errno, "open"); 76 return 1; 77 } 78 79#ifdef CONFIG_POSIX_FALLOCATE 80 if (!td->o.fill_device) { 81 switch (td->o.fallocate_mode) { 82 case FIO_FALLOCATE_NONE: 83 break; 84 case FIO_FALLOCATE_POSIX: 85 dprint(FD_FILE, "posix_fallocate file %s size %llu\n", 86 f->file_name, 87 (unsigned long long) f->real_file_size); 88 89 r = posix_fallocate(f->fd, 0, f->real_file_size); 90 if (r > 0) { 91 log_err("fio: posix_fallocate fails: %s\n", 92 strerror(r)); 93 } 94 break; 95#ifdef CONFIG_LINUX_FALLOCATE 96 case FIO_FALLOCATE_KEEP_SIZE: 97 dprint(FD_FILE, 98 "fallocate(FALLOC_FL_KEEP_SIZE) " 99 "file %s size %llu\n", f->file_name, 100 (unsigned long long) f->real_file_size); 101 102 r = fallocate(f->fd, FALLOC_FL_KEEP_SIZE, 0, 103 f->real_file_size); 104 if (r != 0) 105 td_verror(td, errno, "fallocate"); 106 107 break; 108#endif /* CONFIG_LINUX_FALLOCATE */ 109 default: 110 log_err("fio: unknown fallocate mode: %d\n", 111 td->o.fallocate_mode); 112 assert(0); 113 } 114 } 115#endif /* CONFIG_POSIX_FALLOCATE */ 116 117 if (!new_layout) 118 goto done; 119 120 /* 121 * The size will be -1ULL when fill_device is used, so don't truncate 122 * or fallocate this file, just write it 123 */ 124 if (!td->o.fill_device) { 125 dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name, 126 (unsigned long long) f->real_file_size); 127 if (ftruncate(f->fd, f->real_file_size) == -1) { 128 if (errno != EFBIG) { 129 td_verror(td, errno, "ftruncate"); 130 goto err; 131 } 132 } 133 } 134 135 b = malloc(td->o.max_bs[DDIR_WRITE]); 136 137 left = f->real_file_size; 138 while (left && !td->terminate) { 139 bs = td->o.max_bs[DDIR_WRITE]; 140 if (bs > left) 141 bs = left; 142 143 fill_io_buffer(td, b, bs, bs); 144 145 r = write(f->fd, b, bs); 146 147 if (r > 0) { 148 left -= r; 149 continue; 150 } else { 151 if (r < 0) { 152 int __e = errno; 153 154 if (__e == ENOSPC) { 155 if (td->o.fill_device) 156 break; 157 log_info("fio: ENOSPC on laying out " 158 "file, stopping\n"); 159 break; 160 } 161 td_verror(td, errno, "write"); 162 } else 163 td_verror(td, EIO, "write"); 164 165 break; 166 } 167 } 168 169 if (td->terminate) { 170 dprint(FD_FILE, "terminate unlink %s\n", f->file_name); 171 unlink(f->file_name); 172 } else if (td->o.create_fsync) { 173 if (fsync(f->fd) < 0) { 174 td_verror(td, errno, "fsync"); 175 goto err; 176 } 177 } 178 if (td->o.fill_device && !td_write(td)) { 179 fio_file_clear_size_known(f); 180 if (td_io_get_file_size(td, f)) 181 goto err; 182 if (f->io_size > f->real_file_size) 183 f->io_size = f->real_file_size; 184 } 185 186 free(b); 187done: 188 return 0; 189err: 190 close(f->fd); 191 f->fd = -1; 192 return 1; 193} 194 195static int pre_read_file(struct thread_data *td, struct fio_file *f) 196{ 197 int ret = 0, r, did_open = 0, old_runstate; 198 unsigned long long left; 199 unsigned int bs; 200 char *b; 201 202 if (td->io_ops->flags & FIO_PIPEIO) 203 return 0; 204 205 if (!fio_file_open(f)) { 206 if (td->io_ops->open_file(td, f)) { 207 log_err("fio: cannot pre-read, failed to open file\n"); 208 return 1; 209 } 210 did_open = 1; 211 } 212 213 old_runstate = td_bump_runstate(td, TD_PRE_READING); 214 215 bs = td->o.max_bs[DDIR_READ]; 216 b = malloc(bs); 217 memset(b, 0, bs); 218 219 if (lseek(f->fd, f->file_offset, SEEK_SET) < 0) { 220 td_verror(td, errno, "lseek"); 221 log_err("fio: failed to lseek pre-read file\n"); 222 ret = 1; 223 goto error; 224 } 225 226 left = f->io_size; 227 228 while (left && !td->terminate) { 229 if (bs > left) 230 bs = left; 231 232 r = read(f->fd, b, bs); 233 234 if (r == (int) bs) { 235 left -= bs; 236 continue; 237 } else { 238 td_verror(td, EIO, "pre_read"); 239 break; 240 } 241 } 242 243error: 244 td_restore_runstate(td, old_runstate); 245 246 if (did_open) 247 td->io_ops->close_file(td, f); 248 249 free(b); 250 return ret; 251} 252 253static unsigned long long get_rand_file_size(struct thread_data *td) 254{ 255 unsigned long long ret, sized; 256 unsigned long r; 257 258 if (td->o.use_os_rand) { 259 r = os_random_long(&td->file_size_state); 260 sized = td->o.file_size_high - td->o.file_size_low; 261 ret = (unsigned long long) ((double) sized * (r / (OS_RAND_MAX + 1.0))); 262 } else { 263 r = __rand(&td->__file_size_state); 264 sized = td->o.file_size_high - td->o.file_size_low; 265 ret = (unsigned long long) ((double) sized * (r / (FRAND_MAX + 1.0))); 266 } 267 268 ret += td->o.file_size_low; 269 ret -= (ret % td->o.rw_min_bs); 270 return ret; 271} 272 273static int file_size(struct thread_data *td, struct fio_file *f) 274{ 275 struct stat st; 276 277 if (stat(f->file_name, &st) == -1) { 278 td_verror(td, errno, "fstat"); 279 return 1; 280 } 281 282 f->real_file_size = st.st_size; 283 return 0; 284} 285 286static int bdev_size(struct thread_data *td, struct fio_file *f) 287{ 288 unsigned long long bytes = 0; 289 int r; 290 291 if (td->io_ops->open_file(td, f)) { 292 log_err("fio: failed opening blockdev %s for size check\n", 293 f->file_name); 294 return 1; 295 } 296 297 r = blockdev_size(f, &bytes); 298 if (r) { 299 td_verror(td, r, "blockdev_size"); 300 goto err; 301 } 302 303 if (!bytes) { 304 log_err("%s: zero sized block device?\n", f->file_name); 305 goto err; 306 } 307 308 f->real_file_size = bytes; 309 td->io_ops->close_file(td, f); 310 return 0; 311err: 312 td->io_ops->close_file(td, f); 313 return 1; 314} 315 316static int char_size(struct thread_data *td, struct fio_file *f) 317{ 318#ifdef FIO_HAVE_CHARDEV_SIZE 319 unsigned long long bytes = 0; 320 int r; 321 322 if (td->io_ops->open_file(td, f)) { 323 log_err("fio: failed opening blockdev %s for size check\n", 324 f->file_name); 325 return 1; 326 } 327 328 r = chardev_size(f, &bytes); 329 if (r) { 330 td_verror(td, r, "chardev_size"); 331 goto err; 332 } 333 334 if (!bytes) { 335 log_err("%s: zero sized char device?\n", f->file_name); 336 goto err; 337 } 338 339 f->real_file_size = bytes; 340 td->io_ops->close_file(td, f); 341 return 0; 342err: 343 td->io_ops->close_file(td, f); 344 return 1; 345#else 346 f->real_file_size = -1ULL; 347 return 0; 348#endif 349} 350 351static int get_file_size(struct thread_data *td, struct fio_file *f) 352{ 353 int ret = 0; 354 355 if (fio_file_size_known(f)) 356 return 0; 357 358 if (f->filetype == FIO_TYPE_FILE) 359 ret = file_size(td, f); 360 else if (f->filetype == FIO_TYPE_BD) 361 ret = bdev_size(td, f); 362 else if (f->filetype == FIO_TYPE_CHAR) 363 ret = char_size(td, f); 364 else 365 f->real_file_size = -1; 366 367 if (ret) 368 return ret; 369 370 if (f->file_offset > f->real_file_size) { 371 log_err("%s: offset extends end (%llu > %llu)\n", td->o.name, 372 (unsigned long long) f->file_offset, 373 (unsigned long long) f->real_file_size); 374 return 1; 375 } 376 377 fio_file_set_size_known(f); 378 return 0; 379} 380 381static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f, 382 unsigned long long off, 383 unsigned long long len) 384{ 385 int ret = 0; 386 387 if (len == -1ULL) 388 len = f->io_size; 389 if (off == -1ULL) 390 off = f->file_offset; 391 392 if (len == -1ULL || off == -1ULL) 393 return 0; 394 395 dprint(FD_IO, "invalidate cache %s: %llu/%llu\n", f->file_name, off, 396 len); 397 398 if (f->mmap_ptr) { 399 ret = posix_madvise(f->mmap_ptr, f->mmap_sz, POSIX_MADV_DONTNEED); 400#ifdef FIO_MADV_FREE 401 if (f->filetype == FIO_TYPE_BD) 402 (void) posix_madvise(f->mmap_ptr, f->mmap_sz, FIO_MADV_FREE); 403#endif 404 } else if (f->filetype == FIO_TYPE_FILE) { 405 ret = posix_fadvise(f->fd, off, len, POSIX_FADV_DONTNEED); 406 } else if (f->filetype == FIO_TYPE_BD) { 407 ret = blockdev_invalidate_cache(f); 408 if (ret < 0 && errno == EACCES && geteuid()) { 409 if (!root_warn) { 410 log_err("fio: only root may flush block " 411 "devices. Cache flush bypassed!\n"); 412 root_warn = 1; 413 } 414 ret = 0; 415 } 416 } else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE) 417 ret = 0; 418 419 /* 420 * Cache flushing isn't a fatal condition, and we know it will 421 * happen on some platforms where we don't have the proper 422 * function to flush eg block device caches. So just warn and 423 * continue on our way. 424 */ 425 if (ret) { 426 log_info("fio: cache invalidation of %s failed: %s\n", f->file_name, strerror(errno)); 427 ret = 0; 428 } 429 430 return 0; 431 432} 433 434int file_invalidate_cache(struct thread_data *td, struct fio_file *f) 435{ 436 if (!fio_file_open(f)) 437 return 0; 438 439 return __file_invalidate_cache(td, f, -1ULL, -1ULL); 440} 441 442int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f) 443{ 444 int ret = 0; 445 446 dprint(FD_FILE, "fd close %s\n", f->file_name); 447 448 remove_file_hash(f); 449 450 if (close(f->fd) < 0) 451 ret = errno; 452 453 f->fd = -1; 454 455 if (f->shadow_fd != -1) { 456 close(f->shadow_fd); 457 f->shadow_fd = -1; 458 } 459 460 f->engine_data = 0; 461 return ret; 462} 463 464int file_lookup_open(struct fio_file *f, int flags) 465{ 466 struct fio_file *__f; 467 int from_hash; 468 469 __f = lookup_file_hash(f->file_name); 470 if (__f) { 471 dprint(FD_FILE, "found file in hash %s\n", f->file_name); 472 /* 473 * racy, need the __f->lock locked 474 */ 475 f->lock = __f->lock; 476 from_hash = 1; 477 } else { 478 dprint(FD_FILE, "file not found in hash %s\n", f->file_name); 479 from_hash = 0; 480 } 481 482 f->fd = open(f->file_name, flags, 0600); 483 return from_hash; 484} 485 486static int file_close_shadow_fds(struct thread_data *td) 487{ 488 struct fio_file *f; 489 int num_closed = 0; 490 unsigned int i; 491 492 for_each_file(td, f, i) { 493 if (f->shadow_fd == -1) 494 continue; 495 496 close(f->shadow_fd); 497 f->shadow_fd = -1; 498 num_closed++; 499 } 500 501 return num_closed; 502} 503 504int generic_open_file(struct thread_data *td, struct fio_file *f) 505{ 506 int is_std = 0; 507 int flags = 0; 508 int from_hash = 0; 509 510 dprint(FD_FILE, "fd open %s\n", f->file_name); 511 512 if (td_trim(td) && f->filetype != FIO_TYPE_BD) { 513 log_err("fio: trim only applies to block device\n"); 514 return 1; 515 } 516 517 if (!strcmp(f->file_name, "-")) { 518 if (td_rw(td)) { 519 log_err("fio: can't read/write to stdin/out\n"); 520 return 1; 521 } 522 is_std = 1; 523 524 /* 525 * move output logging to stderr, if we are writing to stdout 526 */ 527 if (td_write(td)) 528 f_out = stderr; 529 } 530 531 if (td_trim(td)) 532 goto skip_flags; 533 if (td->o.odirect) 534 flags |= OS_O_DIRECT; 535 if (td->o.oatomic) { 536 if (!FIO_O_ATOMIC) { 537 td_verror(td, EINVAL, "OS does not support atomic IO"); 538 return 1; 539 } 540 flags |= OS_O_DIRECT | FIO_O_ATOMIC; 541 } 542 if (td->o.sync_io) 543 flags |= O_SYNC; 544 if (td->o.create_on_open) 545 flags |= O_CREAT; 546skip_flags: 547 if (f->filetype != FIO_TYPE_FILE) 548 flags |= FIO_O_NOATIME; 549 550open_again: 551 if (td_write(td)) { 552 if (!read_only) 553 flags |= O_RDWR; 554 555 if (f->filetype == FIO_TYPE_FILE) 556 flags |= O_CREAT; 557 558 if (is_std) 559 f->fd = dup(STDOUT_FILENO); 560 else 561 from_hash = file_lookup_open(f, flags); 562 } else if (td_read(td)) { 563 if (f->filetype == FIO_TYPE_CHAR && !read_only) 564 flags |= O_RDWR; 565 else 566 flags |= O_RDONLY; 567 568 if (is_std) 569 f->fd = dup(STDIN_FILENO); 570 else 571 from_hash = file_lookup_open(f, flags); 572 } else { //td trim 573 flags |= O_RDWR; 574 from_hash = file_lookup_open(f, flags); 575 } 576 577 if (f->fd == -1) { 578 char buf[FIO_VERROR_SIZE]; 579 int __e = errno; 580 581 if (__e == EPERM && (flags & FIO_O_NOATIME)) { 582 flags &= ~FIO_O_NOATIME; 583 goto open_again; 584 } 585 if (__e == EMFILE && file_close_shadow_fds(td)) 586 goto open_again; 587 588 snprintf(buf, sizeof(buf), "open(%s)", f->file_name); 589 590 if (__e == EINVAL && (flags & OS_O_DIRECT)) { 591 log_err("fio: looks like your file system does not " \ 592 "support direct=1/buffered=0\n"); 593 } 594 595 td_verror(td, __e, buf); 596 } 597 598 if (!from_hash && f->fd != -1) { 599 if (add_file_hash(f)) { 600 int fio_unused ret; 601 602 /* 603 * Stash away descriptor for later close. This is to 604 * work-around a "feature" on Linux, where a close of 605 * an fd that has been opened for write will trigger 606 * udev to call blkid to check partitions, fs id, etc. 607 * That pollutes the device cache, which can slow down 608 * unbuffered accesses. 609 */ 610 if (f->shadow_fd == -1) 611 f->shadow_fd = f->fd; 612 else { 613 /* 614 * OK to ignore, we haven't done anything 615 * with it 616 */ 617 ret = generic_close_file(td, f); 618 } 619 goto open_again; 620 } 621 } 622 623 return 0; 624} 625 626int generic_get_file_size(struct thread_data *td, struct fio_file *f) 627{ 628 return get_file_size(td, f); 629} 630 631/* 632 * open/close all files, so that ->real_file_size gets set 633 */ 634static int get_file_sizes(struct thread_data *td) 635{ 636 struct fio_file *f; 637 unsigned int i; 638 int err = 0; 639 640 for_each_file(td, f, i) { 641 dprint(FD_FILE, "get file size for %p/%d/%p\n", f, i, 642 f->file_name); 643 644 if (td_io_get_file_size(td, f)) { 645 if (td->error != ENOENT) { 646 log_err("%s\n", td->verror); 647 err = 1; 648 } 649 clear_error(td); 650 } 651 652 if (f->real_file_size == -1ULL && td->o.size) 653 f->real_file_size = td->o.size / td->o.nr_files; 654 } 655 656 return err; 657} 658 659struct fio_mount { 660 struct flist_head list; 661 const char *base; 662 char __base[256]; 663 unsigned int key; 664}; 665 666/* 667 * Get free number of bytes for each file on each unique mount. 668 */ 669static unsigned long long get_fs_free_counts(struct thread_data *td) 670{ 671 struct flist_head *n, *tmp; 672 unsigned long long ret = 0; 673 struct fio_mount *fm; 674 FLIST_HEAD(list); 675 struct fio_file *f; 676 unsigned int i; 677 678 for_each_file(td, f, i) { 679 struct stat sb; 680 char buf[256]; 681 682 if (f->filetype == FIO_TYPE_BD || f->filetype == FIO_TYPE_CHAR) { 683 if (f->real_file_size != -1ULL) 684 ret += f->real_file_size; 685 continue; 686 } else if (f->filetype != FIO_TYPE_FILE) 687 continue; 688 689 strcpy(buf, f->file_name); 690 691 if (stat(buf, &sb) < 0) { 692 if (errno != ENOENT) 693 break; 694 strcpy(buf, "."); 695 if (stat(buf, &sb) < 0) 696 break; 697 } 698 699 fm = NULL; 700 flist_for_each(n, &list) { 701 fm = flist_entry(n, struct fio_mount, list); 702 if (fm->key == sb.st_dev) 703 break; 704 705 fm = NULL; 706 } 707 708 if (fm) 709 continue; 710 711 fm = malloc(sizeof(*fm)); 712 strcpy(fm->__base, buf); 713 fm->base = basename(fm->__base); 714 fm->key = sb.st_dev; 715 flist_add(&fm->list, &list); 716 } 717 718 flist_for_each_safe(n, tmp, &list) { 719 unsigned long long sz; 720 721 fm = flist_entry(n, struct fio_mount, list); 722 flist_del(&fm->list); 723 724 sz = get_fs_size(fm->base); 725 if (sz && sz != -1ULL) 726 ret += sz; 727 728 free(fm); 729 } 730 731 return ret; 732} 733 734uint64_t get_start_offset(struct thread_data *td, struct fio_file *f) 735{ 736 struct thread_options *o = &td->o; 737 738 if (o->file_append && f->filetype == FIO_TYPE_FILE) 739 return f->real_file_size; 740 741 return td->o.start_offset + 742 (td->thread_number - 1) * td->o.offset_increment; 743} 744 745/* 746 * Open the files and setup files sizes, creating files if necessary. 747 */ 748int setup_files(struct thread_data *td) 749{ 750 unsigned long long total_size, extend_size; 751 struct thread_options *o = &td->o; 752 struct fio_file *f; 753 unsigned int i, nr_fs_extra = 0; 754 int err = 0, need_extend; 755 int old_state; 756 const unsigned int bs = td_min_bs(td); 757 uint64_t fs = 0; 758 759 dprint(FD_FILE, "setup files\n"); 760 761 old_state = td_bump_runstate(td, TD_SETTING_UP); 762 763 if (o->read_iolog_file) 764 goto done; 765 766 /* 767 * if ioengine defines a setup() method, it's responsible for 768 * opening the files and setting f->real_file_size to indicate 769 * the valid range for that file. 770 */ 771 if (td->io_ops->setup) 772 err = td->io_ops->setup(td); 773 else 774 err = get_file_sizes(td); 775 776 if (err) 777 goto err_out; 778 779 /* 780 * check sizes. if the files/devices do not exist and the size 781 * isn't passed to fio, abort. 782 */ 783 total_size = 0; 784 for_each_file(td, f, i) { 785 if (f->real_file_size == -1ULL) 786 total_size = -1ULL; 787 else 788 total_size += f->real_file_size; 789 } 790 791 if (o->fill_device) 792 td->fill_device_size = get_fs_free_counts(td); 793 794 /* 795 * device/file sizes are zero and no size given, punt 796 */ 797 if ((!total_size || total_size == -1ULL) && !o->size && 798 !(td->io_ops->flags & FIO_NOIO) && !o->fill_device && 799 !(o->nr_files && (o->file_size_low || o->file_size_high))) { 800 log_err("%s: you need to specify size=\n", o->name); 801 td_verror(td, EINVAL, "total_file_size"); 802 goto err_out; 803 } 804 805 /* 806 * Calculate per-file size and potential extra size for the 807 * first files, if needed. 808 */ 809 if (!o->file_size_low && o->nr_files) { 810 uint64_t all_fs; 811 812 fs = o->size / o->nr_files; 813 all_fs = fs * o->nr_files; 814 815 if (all_fs < o->size) 816 nr_fs_extra = (o->size - all_fs) / bs; 817 } 818 819 /* 820 * now file sizes are known, so we can set ->io_size. if size= is 821 * not given, ->io_size is just equal to ->real_file_size. if size 822 * is given, ->io_size is size / nr_files. 823 */ 824 extend_size = total_size = 0; 825 need_extend = 0; 826 for_each_file(td, f, i) { 827 f->file_offset = get_start_offset(td, f); 828 829 if (!o->file_size_low) { 830 /* 831 * no file size range given, file size is equal to 832 * total size divided by number of files. If that is 833 * zero, set it to the real file size. If the size 834 * doesn't divide nicely with the min blocksize, 835 * make the first files bigger. 836 */ 837 f->io_size = fs; 838 if (nr_fs_extra) { 839 nr_fs_extra--; 840 f->io_size += bs; 841 } 842 843 if (!f->io_size) 844 f->io_size = f->real_file_size - f->file_offset; 845 } else if (f->real_file_size < o->file_size_low || 846 f->real_file_size > o->file_size_high) { 847 if (f->file_offset > o->file_size_low) 848 goto err_offset; 849 /* 850 * file size given. if it's fixed, use that. if it's a 851 * range, generate a random size in-between. 852 */ 853 if (o->file_size_low == o->file_size_high) 854 f->io_size = o->file_size_low - f->file_offset; 855 else { 856 f->io_size = get_rand_file_size(td) 857 - f->file_offset; 858 } 859 } else 860 f->io_size = f->real_file_size - f->file_offset; 861 862 if (f->io_size == -1ULL) 863 total_size = -1ULL; 864 else { 865 if (o->size_percent) 866 f->io_size = (f->io_size * o->size_percent) / 100; 867 total_size += f->io_size; 868 } 869 870 if (f->filetype == FIO_TYPE_FILE && 871 (f->io_size + f->file_offset) > f->real_file_size && 872 !(td->io_ops->flags & FIO_DISKLESSIO)) { 873 if (!o->create_on_open) { 874 need_extend++; 875 extend_size += (f->io_size + f->file_offset); 876 } else 877 f->real_file_size = f->io_size + f->file_offset; 878 fio_file_set_extend(f); 879 } 880 } 881 882 if (!o->size || o->size > total_size) 883 o->size = total_size; 884 885 /* 886 * See if we need to extend some files 887 */ 888 if (need_extend) { 889 temp_stall_ts = 1; 890 if (output_format == FIO_OUTPUT_NORMAL) 891 log_info("%s: Laying out IO file(s) (%u file(s) /" 892 " %lluMB)\n", o->name, need_extend, 893 extend_size >> 20); 894 895 for_each_file(td, f, i) { 896 unsigned long long old_len = -1ULL, extend_len = -1ULL; 897 898 if (!fio_file_extend(f)) 899 continue; 900 901 assert(f->filetype == FIO_TYPE_FILE); 902 fio_file_clear_extend(f); 903 if (!o->fill_device) { 904 old_len = f->real_file_size; 905 extend_len = f->io_size + f->file_offset - 906 old_len; 907 } 908 f->real_file_size = (f->io_size + f->file_offset); 909 err = extend_file(td, f); 910 if (err) 911 break; 912 913 err = __file_invalidate_cache(td, f, old_len, 914 extend_len); 915 close(f->fd); 916 f->fd = -1; 917 if (err) 918 break; 919 } 920 temp_stall_ts = 0; 921 } 922 923 if (err) 924 goto err_out; 925 926 if (!o->zone_size) 927 o->zone_size = o->size; 928 929 /* 930 * iolog already set the total io size, if we read back 931 * stored entries. 932 */ 933 if (!o->read_iolog_file) 934 td->total_io_size = o->size * o->loops; 935 936done: 937 if (o->create_only) 938 td->done = 1; 939 940 td_restore_runstate(td, old_state); 941 return 0; 942err_offset: 943 log_err("%s: you need to specify valid offset=\n", o->name); 944err_out: 945 td_restore_runstate(td, old_state); 946 return 1; 947} 948 949int pre_read_files(struct thread_data *td) 950{ 951 struct fio_file *f; 952 unsigned int i; 953 954 dprint(FD_FILE, "pre_read files\n"); 955 956 for_each_file(td, f, i) { 957 pre_read_file(td, f); 958 } 959 960 return 1; 961} 962 963static int __init_rand_distribution(struct thread_data *td, struct fio_file *f) 964{ 965 unsigned int range_size, seed; 966 unsigned long nranges; 967 uint64_t file_size; 968 969 range_size = min(td->o.min_bs[DDIR_READ], td->o.min_bs[DDIR_WRITE]); 970 file_size = min(f->real_file_size, f->io_size); 971 972 nranges = (file_size + range_size - 1) / range_size; 973 974 seed = jhash(f->file_name, strlen(f->file_name), 0) * td->thread_number; 975 if (!td->o.rand_repeatable) 976 seed = td->rand_seeds[4]; 977 978 if (td->o.random_distribution == FIO_RAND_DIST_ZIPF) 979 zipf_init(&f->zipf, nranges, td->o.zipf_theta.u.f, seed); 980 else 981 pareto_init(&f->zipf, nranges, td->o.pareto_h.u.f, seed); 982 983 return 1; 984} 985 986static int init_rand_distribution(struct thread_data *td) 987{ 988 struct fio_file *f; 989 unsigned int i; 990 int state; 991 992 if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) 993 return 0; 994 995 state = td_bump_runstate(td, TD_SETTING_UP); 996 997 for_each_file(td, f, i) 998 __init_rand_distribution(td, f); 999 1000 td_restore_runstate(td, state); 1001 1002 return 1; 1003} 1004 1005int init_random_map(struct thread_data *td) 1006{ 1007 unsigned long long blocks; 1008 struct fio_file *f; 1009 unsigned int i; 1010 1011 if (init_rand_distribution(td)) 1012 return 0; 1013 if (!td_random(td)) 1014 return 0; 1015 1016 for_each_file(td, f, i) { 1017 uint64_t file_size = min(f->real_file_size, f->io_size); 1018 1019 blocks = file_size / (unsigned long long) td->o.rw_min_bs; 1020 1021 if (td->o.random_generator == FIO_RAND_GEN_LFSR) { 1022 unsigned long seed; 1023 1024 seed = td->rand_seeds[FIO_RAND_BLOCK_OFF]; 1025 1026 if (!lfsr_init(&f->lfsr, blocks, seed, 0)) 1027 continue; 1028 } else if (!td->o.norandommap) { 1029 f->io_axmap = axmap_new(blocks); 1030 if (f->io_axmap) 1031 continue; 1032 } else if (td->o.norandommap) 1033 continue; 1034 1035 if (!td->o.softrandommap) { 1036 log_err("fio: failed allocating random map. If running" 1037 " a large number of jobs, try the 'norandommap'" 1038 " option or set 'softrandommap'. Or give" 1039 " a larger --alloc-size to fio.\n"); 1040 return 1; 1041 } 1042 1043 log_info("fio: file %s failed allocating random map. Running " 1044 "job without.\n", f->file_name); 1045 } 1046 1047 return 0; 1048} 1049 1050void close_files(struct thread_data *td) 1051{ 1052 struct fio_file *f; 1053 unsigned int i; 1054 1055 for_each_file(td, f, i) { 1056 if (fio_file_open(f)) 1057 td_io_close_file(td, f); 1058 } 1059} 1060 1061void close_and_free_files(struct thread_data *td) 1062{ 1063 struct fio_file *f; 1064 unsigned int i; 1065 1066 dprint(FD_FILE, "close files\n"); 1067 1068 for_each_file(td, f, i) { 1069 if (fio_file_open(f)) 1070 td_io_close_file(td, f); 1071 1072 remove_file_hash(f); 1073 1074 if (td->o.unlink && f->filetype == FIO_TYPE_FILE) { 1075 dprint(FD_FILE, "free unlink %s\n", f->file_name); 1076 unlink(f->file_name); 1077 } 1078 1079 sfree(f->file_name); 1080 f->file_name = NULL; 1081 axmap_free(f->io_axmap); 1082 f->io_axmap = NULL; 1083 sfree(f); 1084 } 1085 1086 td->o.filename = NULL; 1087 free(td->files); 1088 free(td->file_locks); 1089 td->files_index = 0; 1090 td->files = NULL; 1091 td->file_locks = NULL; 1092 td->o.file_lock_mode = FILE_LOCK_NONE; 1093 td->o.nr_files = 0; 1094} 1095 1096static void get_file_type(struct fio_file *f) 1097{ 1098 struct stat sb; 1099 1100 if (!strcmp(f->file_name, "-")) 1101 f->filetype = FIO_TYPE_PIPE; 1102 else 1103 f->filetype = FIO_TYPE_FILE; 1104 1105 /* \\.\ is the device namespace in Windows, where every file is 1106 * a block device */ 1107 if (strncmp(f->file_name, "\\\\.\\", 4) == 0) 1108 f->filetype = FIO_TYPE_BD; 1109 1110 if (!stat(f->file_name, &sb)) { 1111 if (S_ISBLK(sb.st_mode)) 1112 f->filetype = FIO_TYPE_BD; 1113 else if (S_ISCHR(sb.st_mode)) 1114 f->filetype = FIO_TYPE_CHAR; 1115 else if (S_ISFIFO(sb.st_mode)) 1116 f->filetype = FIO_TYPE_PIPE; 1117 } 1118} 1119 1120static int __is_already_allocated(const char *fname) 1121{ 1122 struct flist_head *entry; 1123 char *filename; 1124 1125 if (flist_empty(&filename_list)) 1126 return 0; 1127 1128 flist_for_each(entry, &filename_list) { 1129 filename = flist_entry(entry, struct file_name, list)->filename; 1130 1131 if (strcmp(filename, fname) == 0) 1132 return 1; 1133 } 1134 1135 return 0; 1136} 1137 1138static int is_already_allocated(const char *fname) 1139{ 1140 int ret; 1141 1142 fio_file_hash_lock(); 1143 ret = __is_already_allocated(fname); 1144 fio_file_hash_unlock(); 1145 return ret; 1146} 1147 1148static void set_already_allocated(const char *fname) 1149{ 1150 struct file_name *fn; 1151 1152 fn = malloc(sizeof(struct file_name)); 1153 fn->filename = strdup(fname); 1154 1155 fio_file_hash_lock(); 1156 if (!__is_already_allocated(fname)) { 1157 flist_add_tail(&fn->list, &filename_list); 1158 fn = NULL; 1159 } 1160 fio_file_hash_unlock(); 1161 1162 if (fn) { 1163 free(fn->filename); 1164 free(fn); 1165 } 1166} 1167 1168 1169static void free_already_allocated(void) 1170{ 1171 struct flist_head *entry, *tmp; 1172 struct file_name *fn; 1173 1174 if (flist_empty(&filename_list)) 1175 return; 1176 1177 fio_file_hash_lock(); 1178 flist_for_each_safe(entry, tmp, &filename_list) { 1179 fn = flist_entry(entry, struct file_name, list); 1180 free(fn->filename); 1181 flist_del(&fn->list); 1182 free(fn); 1183 } 1184 1185 fio_file_hash_unlock(); 1186} 1187 1188static struct fio_file *alloc_new_file(struct thread_data *td) 1189{ 1190 struct fio_file *f; 1191 1192 f = smalloc(sizeof(*f)); 1193 if (!f) { 1194 log_err("fio: smalloc OOM\n"); 1195 assert(0); 1196 return NULL; 1197 } 1198 1199 f->fd = -1; 1200 f->shadow_fd = -1; 1201 fio_file_reset(td, f); 1202 return f; 1203} 1204 1205int add_file(struct thread_data *td, const char *fname, int numjob, int inc) 1206{ 1207 int cur_files = td->files_index; 1208 char file_name[PATH_MAX]; 1209 struct fio_file *f; 1210 int len = 0; 1211 1212 dprint(FD_FILE, "add file %s\n", fname); 1213 1214 if (td->o.directory) 1215 len = set_name_idx(file_name, td->o.directory, numjob); 1216 1217 sprintf(file_name + len, "%s", fname); 1218 1219 /* clean cloned siblings using existing files */ 1220 if (numjob && is_already_allocated(file_name)) 1221 return 0; 1222 1223 f = alloc_new_file(td); 1224 1225 if (td->files_size <= td->files_index) { 1226 unsigned int new_size = td->o.nr_files + 1; 1227 1228 dprint(FD_FILE, "resize file array to %d files\n", new_size); 1229 1230 td->files = realloc(td->files, new_size * sizeof(f)); 1231 if (td->files == NULL) { 1232 log_err("fio: realloc OOM\n"); 1233 assert(0); 1234 } 1235 if (td->o.file_lock_mode != FILE_LOCK_NONE) { 1236 td->file_locks = realloc(td->file_locks, new_size); 1237 if (!td->file_locks) { 1238 log_err("fio: realloc OOM\n"); 1239 assert(0); 1240 } 1241 td->file_locks[cur_files] = FILE_LOCK_NONE; 1242 } 1243 td->files_size = new_size; 1244 } 1245 td->files[cur_files] = f; 1246 f->fileno = cur_files; 1247 1248 /* 1249 * init function, io engine may not be loaded yet 1250 */ 1251 if (td->io_ops && (td->io_ops->flags & FIO_DISKLESSIO)) 1252 f->real_file_size = -1ULL; 1253 1254 f->file_name = smalloc_strdup(file_name); 1255 if (!f->file_name) { 1256 log_err("fio: smalloc OOM\n"); 1257 assert(0); 1258 } 1259 1260 get_file_type(f); 1261 1262 switch (td->o.file_lock_mode) { 1263 case FILE_LOCK_NONE: 1264 break; 1265 case FILE_LOCK_READWRITE: 1266 f->rwlock = fio_rwlock_init(); 1267 break; 1268 case FILE_LOCK_EXCLUSIVE: 1269 f->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED); 1270 break; 1271 default: 1272 log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode); 1273 assert(0); 1274 } 1275 1276 td->files_index++; 1277 if (f->filetype == FIO_TYPE_FILE) 1278 td->nr_normal_files++; 1279 1280 set_already_allocated(file_name); 1281 1282 if (inc) 1283 td->o.nr_files++; 1284 1285 dprint(FD_FILE, "file %p \"%s\" added at %d\n", f, f->file_name, 1286 cur_files); 1287 1288 return cur_files; 1289} 1290 1291int add_file_exclusive(struct thread_data *td, const char *fname) 1292{ 1293 struct fio_file *f; 1294 unsigned int i; 1295 1296 for_each_file(td, f, i) { 1297 if (!strcmp(f->file_name, fname)) 1298 return i; 1299 } 1300 1301 return add_file(td, fname, 0, 1); 1302} 1303 1304void get_file(struct fio_file *f) 1305{ 1306 dprint(FD_FILE, "get file %s, ref=%d\n", f->file_name, f->references); 1307 assert(fio_file_open(f)); 1308 f->references++; 1309} 1310 1311int put_file(struct thread_data *td, struct fio_file *f) 1312{ 1313 int f_ret = 0, ret = 0; 1314 1315 dprint(FD_FILE, "put file %s, ref=%d\n", f->file_name, f->references); 1316 1317 if (!fio_file_open(f)) { 1318 assert(f->fd == -1); 1319 return 0; 1320 } 1321 1322 assert(f->references); 1323 if (--f->references) 1324 return 0; 1325 1326 if (should_fsync(td) && td->o.fsync_on_close) 1327 f_ret = fsync(f->fd); 1328 1329 if (td->io_ops->close_file) 1330 ret = td->io_ops->close_file(td, f); 1331 1332 if (!ret) 1333 ret = f_ret; 1334 1335 td->nr_open_files--; 1336 fio_file_clear_open(f); 1337 assert(f->fd == -1); 1338 return ret; 1339} 1340 1341void lock_file(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir) 1342{ 1343 if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE) 1344 return; 1345 1346 if (td->o.file_lock_mode == FILE_LOCK_READWRITE) { 1347 if (ddir == DDIR_READ) 1348 fio_rwlock_read(f->rwlock); 1349 else 1350 fio_rwlock_write(f->rwlock); 1351 } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) 1352 fio_mutex_down(f->lock); 1353 1354 td->file_locks[f->fileno] = td->o.file_lock_mode; 1355} 1356 1357void unlock_file(struct thread_data *td, struct fio_file *f) 1358{ 1359 if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE) 1360 return; 1361 1362 if (td->o.file_lock_mode == FILE_LOCK_READWRITE) 1363 fio_rwlock_unlock(f->rwlock); 1364 else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) 1365 fio_mutex_up(f->lock); 1366 1367 td->file_locks[f->fileno] = FILE_LOCK_NONE; 1368} 1369 1370void unlock_file_all(struct thread_data *td, struct fio_file *f) 1371{ 1372 if (td->o.file_lock_mode == FILE_LOCK_NONE || !td->file_locks) 1373 return; 1374 if (td->file_locks[f->fileno] != FILE_LOCK_NONE) 1375 unlock_file(td, f); 1376} 1377 1378static int recurse_dir(struct thread_data *td, const char *dirname) 1379{ 1380 struct dirent *dir; 1381 int ret = 0; 1382 DIR *D; 1383 1384 D = opendir(dirname); 1385 if (!D) { 1386 char buf[FIO_VERROR_SIZE]; 1387 1388 snprintf(buf, FIO_VERROR_SIZE, "opendir(%s)", dirname); 1389 td_verror(td, errno, buf); 1390 return 1; 1391 } 1392 1393 while ((dir = readdir(D)) != NULL) { 1394 char full_path[PATH_MAX]; 1395 struct stat sb; 1396 1397 if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) 1398 continue; 1399 1400 sprintf(full_path, "%s%s%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name); 1401 1402 if (lstat(full_path, &sb) == -1) { 1403 if (errno != ENOENT) { 1404 td_verror(td, errno, "stat"); 1405 return 1; 1406 } 1407 } 1408 1409 if (S_ISREG(sb.st_mode)) { 1410 add_file(td, full_path, 0, 1); 1411 continue; 1412 } 1413 if (!S_ISDIR(sb.st_mode)) 1414 continue; 1415 1416 ret = recurse_dir(td, full_path); 1417 if (ret) 1418 break; 1419 } 1420 1421 closedir(D); 1422 return ret; 1423} 1424 1425int add_dir_files(struct thread_data *td, const char *path) 1426{ 1427 int ret = recurse_dir(td, path); 1428 1429 if (!ret) 1430 log_info("fio: opendir added %d files\n", td->o.nr_files); 1431 1432 return ret; 1433} 1434 1435void dup_files(struct thread_data *td, struct thread_data *org) 1436{ 1437 struct fio_file *f; 1438 unsigned int i; 1439 1440 dprint(FD_FILE, "dup files: %d\n", org->files_index); 1441 1442 if (!org->files) 1443 return; 1444 1445 td->files = malloc(org->files_index * sizeof(f)); 1446 1447 if (td->o.file_lock_mode != FILE_LOCK_NONE) 1448 td->file_locks = malloc(org->files_index); 1449 1450 for_each_file(org, f, i) { 1451 struct fio_file *__f; 1452 1453 __f = alloc_new_file(td); 1454 1455 if (f->file_name) { 1456 __f->file_name = smalloc_strdup(f->file_name); 1457 if (!__f->file_name) { 1458 log_err("fio: smalloc OOM\n"); 1459 assert(0); 1460 } 1461 1462 __f->filetype = f->filetype; 1463 } 1464 1465 if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) 1466 __f->lock = f->lock; 1467 else if (td->o.file_lock_mode == FILE_LOCK_READWRITE) 1468 __f->rwlock = f->rwlock; 1469 1470 td->files[i] = __f; 1471 } 1472} 1473 1474/* 1475 * Returns the index that matches the filename, or -1 if not there 1476 */ 1477int get_fileno(struct thread_data *td, const char *fname) 1478{ 1479 struct fio_file *f; 1480 unsigned int i; 1481 1482 for_each_file(td, f, i) 1483 if (!strcmp(f->file_name, fname)) 1484 return i; 1485 1486 return -1; 1487} 1488 1489/* 1490 * For log usage, where we add/open/close files automatically 1491 */ 1492void free_release_files(struct thread_data *td) 1493{ 1494 close_files(td); 1495 td->files_index = 0; 1496 td->nr_normal_files = 0; 1497} 1498 1499void fio_file_reset(struct thread_data *td, struct fio_file *f) 1500{ 1501 f->last_pos = f->file_offset; 1502 f->last_start = -1ULL; 1503 if (f->io_axmap) 1504 axmap_reset(f->io_axmap); 1505 if (td->o.random_generator == FIO_RAND_GEN_LFSR) 1506 lfsr_reset(&f->lfsr, td->rand_seeds[FIO_RAND_BLOCK_OFF]); 1507} 1508 1509int fio_files_done(struct thread_data *td) 1510{ 1511 struct fio_file *f; 1512 unsigned int i; 1513 1514 for_each_file(td, f, i) 1515 if (!fio_file_done(f)) 1516 return 0; 1517 1518 return 1; 1519} 1520 1521/* free memory used in initialization phase only */ 1522void filesetup_mem_free(void) 1523{ 1524 free_already_allocated(); 1525} 1526