verify.c revision 3ebcfd56868fd11f66bee3a9792811c02d66db04
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9#include <libgen.h> 10 11#include "fio.h" 12#include "verify.h" 13#include "trim.h" 14#include "lib/rand.h" 15#include "lib/hweight.h" 16 17#include "crc/md5.h" 18#include "crc/crc64.h" 19#include "crc/crc32.h" 20#include "crc/crc32c.h" 21#include "crc/crc16.h" 22#include "crc/crc7.h" 23#include "crc/sha256.h" 24#include "crc/sha512.h" 25#include "crc/sha1.h" 26#include "crc/xxhash.h" 27 28static void populate_hdr(struct thread_data *td, struct io_u *io_u, 29 struct verify_header *hdr, unsigned int header_num, 30 unsigned int header_len); 31 32static void fill_pattern(struct thread_data *td, void *p, unsigned int len, 33 char *pattern, unsigned int pattern_bytes) 34{ 35 switch (pattern_bytes) { 36 case 0: 37 assert(0); 38 break; 39 case 1: 40 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 41 memset(p, pattern[0], len); 42 break; 43 default: { 44 unsigned int i = 0, size = 0; 45 unsigned char *b = p; 46 47 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 48 pattern_bytes, len); 49 50 while (i < len) { 51 size = pattern_bytes; 52 if (size > (len - i)) 53 size = len - i; 54 memcpy(b+i, pattern, size); 55 i += size; 56 } 57 break; 58 } 59 } 60} 61 62void fill_buffer_pattern(struct thread_data *td, void *p, unsigned int len) 63{ 64 fill_pattern(td, p, len, td->o.buffer_pattern, td->o.buffer_pattern_bytes); 65} 66 67void fill_verify_pattern(struct thread_data *td, void *p, unsigned int len, 68 struct io_u *io_u, unsigned long seed, int use_seed) 69{ 70 if (!td->o.verify_pattern_bytes) { 71 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 72 73 if (use_seed) 74 __fill_random_buf(p, len, seed); 75 else 76 io_u->rand_seed = fill_random_buf(&td->verify_state, p, len); 77 return; 78 } 79 80 if (io_u->buf_filled_len >= len) { 81 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 82 td->o.verify_pattern_bytes, len); 83 return; 84 } 85 86 fill_pattern(td, p, len, td->o.verify_pattern, td->o.verify_pattern_bytes); 87 88 io_u->buf_filled_len = len; 89} 90 91static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u) 92{ 93 unsigned int hdr_inc; 94 95 hdr_inc = io_u->buflen; 96 if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen) 97 hdr_inc = td->o.verify_interval; 98 99 return hdr_inc; 100} 101 102static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 103 unsigned long seed, int use_seed) 104{ 105 unsigned int hdr_inc, header_num; 106 struct verify_header *hdr; 107 void *p = io_u->buf; 108 109 fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 110 111 hdr_inc = get_hdr_inc(td, io_u); 112 header_num = 0; 113 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 114 hdr = p; 115 populate_hdr(td, io_u, hdr, header_num, hdr_inc); 116 header_num++; 117 } 118} 119 120static void memswp(void *buf1, void *buf2, unsigned int len) 121{ 122 char swap[200]; 123 124 assert(len <= sizeof(swap)); 125 126 memcpy(&swap, buf1, len); 127 memcpy(buf1, buf2, len); 128 memcpy(buf2, &swap, len); 129} 130 131static void hexdump(void *buffer, int len) 132{ 133 unsigned char *p = buffer; 134 int i; 135 136 for (i = 0; i < len; i++) 137 log_err("%02x", p[i]); 138 log_err("\n"); 139} 140 141/* 142 * Prepare for separation of verify_header and checksum header 143 */ 144static inline unsigned int __hdr_size(int verify_type) 145{ 146 unsigned int len = 0; 147 148 switch (verify_type) { 149 case VERIFY_NONE: 150 case VERIFY_NULL: 151 len = 0; 152 break; 153 case VERIFY_MD5: 154 len = sizeof(struct vhdr_md5); 155 break; 156 case VERIFY_CRC64: 157 len = sizeof(struct vhdr_crc64); 158 break; 159 case VERIFY_CRC32C: 160 case VERIFY_CRC32: 161 case VERIFY_CRC32C_INTEL: 162 len = sizeof(struct vhdr_crc32); 163 break; 164 case VERIFY_CRC16: 165 len = sizeof(struct vhdr_crc16); 166 break; 167 case VERIFY_CRC7: 168 len = sizeof(struct vhdr_crc7); 169 break; 170 case VERIFY_SHA256: 171 len = sizeof(struct vhdr_sha256); 172 break; 173 case VERIFY_SHA512: 174 len = sizeof(struct vhdr_sha512); 175 break; 176 case VERIFY_XXHASH: 177 len = sizeof(struct vhdr_xxhash); 178 break; 179 case VERIFY_META: 180 len = sizeof(struct vhdr_meta); 181 break; 182 case VERIFY_SHA1: 183 len = sizeof(struct vhdr_sha1); 184 break; 185 case VERIFY_PATTERN: 186 len = 0; 187 break; 188 default: 189 log_err("fio: unknown verify header!\n"); 190 assert(0); 191 } 192 193 return len + sizeof(struct verify_header); 194} 195 196static inline unsigned int hdr_size(struct verify_header *hdr) 197{ 198 return __hdr_size(hdr->verify_type); 199} 200 201static void *hdr_priv(struct verify_header *hdr) 202{ 203 void *priv = hdr; 204 205 return priv + sizeof(struct verify_header); 206} 207 208/* 209 * Verify container, pass info to verify handlers and allow them to 210 * pass info back in case of error 211 */ 212struct vcont { 213 /* 214 * Input 215 */ 216 struct io_u *io_u; 217 unsigned int hdr_num; 218 struct thread_data *td; 219 220 /* 221 * Output, only valid in case of error 222 */ 223 const char *name; 224 void *good_crc; 225 void *bad_crc; 226 unsigned int crc_len; 227}; 228 229#define DUMP_BUF_SZ 255 230static int dump_buf_warned; 231 232static void dump_buf(char *buf, unsigned int len, unsigned long long offset, 233 const char *type, struct fio_file *f) 234{ 235 char *ptr, fname[DUMP_BUF_SZ]; 236 size_t buf_left = DUMP_BUF_SZ; 237 int ret, fd; 238 239 ptr = strdup(f->file_name); 240 241 fname[DUMP_BUF_SZ - 1] = '\0'; 242 strncpy(fname, basename(ptr), DUMP_BUF_SZ - 1); 243 244 buf_left -= strlen(fname); 245 if (buf_left <= 0) { 246 if (!dump_buf_warned) { 247 log_err("fio: verify failure dump buffer too small\n"); 248 dump_buf_warned = 1; 249 } 250 free(ptr); 251 return; 252 } 253 254 snprintf(fname + strlen(fname), buf_left, ".%llu.%s", offset, type); 255 256 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 257 if (fd < 0) { 258 perror("open verify buf file"); 259 return; 260 } 261 262 while (len) { 263 ret = write(fd, buf, len); 264 if (!ret) 265 break; 266 else if (ret < 0) { 267 perror("write verify buf file"); 268 break; 269 } 270 len -= ret; 271 buf += ret; 272 } 273 274 close(fd); 275 log_err(" %s data dumped as %s\n", type, fname); 276 free(ptr); 277} 278 279/* 280 * Dump the contents of the read block and re-generate the correct data 281 * and dump that too. 282 */ 283static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 284{ 285 struct thread_data *td = vc->td; 286 struct io_u *io_u = vc->io_u; 287 unsigned long hdr_offset; 288 struct io_u dummy; 289 void *buf; 290 291 if (!td->o.verify_dump) 292 return; 293 294 /* 295 * Dump the contents we just read off disk 296 */ 297 hdr_offset = vc->hdr_num * hdr->len; 298 299 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 300 "received", vc->io_u->file); 301 302 /* 303 * Allocate a new buf and re-generate the original data 304 */ 305 buf = malloc(io_u->buflen); 306 dummy = *io_u; 307 dummy.buf = buf; 308 dummy.rand_seed = hdr->rand_seed; 309 dummy.buf_filled_len = 0; 310 dummy.buflen = io_u->buflen; 311 312 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 313 314 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 315 "expected", vc->io_u->file); 316 free(buf); 317} 318 319static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 320{ 321 unsigned long long offset; 322 323 offset = vc->io_u->offset; 324 offset += vc->hdr_num * hdr->len; 325 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 326 vc->name, vc->io_u->file->file_name, offset, hdr->len); 327 328 if (vc->good_crc && vc->bad_crc) { 329 log_err(" Expected CRC: "); 330 hexdump(vc->good_crc, vc->crc_len); 331 log_err(" Received CRC: "); 332 hexdump(vc->bad_crc, vc->crc_len); 333 } 334 335 dump_verify_buffers(hdr, vc); 336} 337 338/* 339 * Return data area 'header_num' 340 */ 341static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 342{ 343 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 344} 345 346static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc) 347{ 348 struct thread_data *td = vc->td; 349 struct io_u *io_u = vc->io_u; 350 char *buf, *pattern; 351 unsigned int header_size = __hdr_size(td->o.verify); 352 unsigned int len, mod, i, size, pattern_size; 353 354 pattern = td->o.verify_pattern; 355 pattern_size = td->o.verify_pattern_bytes; 356 if (pattern_size <= 1) 357 pattern_size = MAX_PATTERN_SIZE; 358 buf = (void *) hdr + header_size; 359 len = get_hdr_inc(td, io_u) - header_size; 360 mod = header_size % pattern_size; 361 362 for (i = 0; i < len; i += size) { 363 size = pattern_size - mod; 364 if (size > (len - i)) 365 size = len - i; 366 if (memcmp(buf + i, pattern + mod, size)) 367 /* Let the slow compare find the first mismatch byte. */ 368 break; 369 mod = 0; 370 } 371 372 for (; i < len; i++) { 373 if (buf[i] != pattern[mod]) { 374 unsigned int bits; 375 376 bits = hweight8(buf[i] ^ pattern[mod]); 377 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 378 buf[i], pattern[mod], bits); 379 log_err("fio: bad pattern block offset %u\n", i); 380 dump_verify_buffers(hdr, vc); 381 return EILSEQ; 382 } 383 mod++; 384 if (mod == td->o.verify_pattern_bytes) 385 mod = 0; 386 } 387 388 return 0; 389} 390 391static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc) 392{ 393 struct thread_data *td = vc->td; 394 struct vhdr_meta *vh = hdr_priv(hdr); 395 struct io_u *io_u = vc->io_u; 396 int ret = EILSEQ; 397 398 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 399 400 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 401 ret = 0; 402 403 if (td->o.verify_pattern_bytes) 404 ret |= verify_io_u_pattern(hdr, vc); 405 406 /* 407 * For read-only workloads, the program cannot be certain of the 408 * last numberio written to a block. Checking of numberio will be 409 * done only for workloads that write data. For verify_only, 410 * numberio will be checked in the last iteration when the correct 411 * state of numberio, that would have been written to each block 412 * in a previous run of fio, has been reached. 413 */ 414 if ((td_write(td) || td_rw(td)) && (td_min_bs(td) == td_max_bs(td)) && 415 !td->o.time_based) 416 if (!td->o.verify_only || td->o.loops == 0) 417 if (vh->numberio != io_u->numberio) 418 ret = EILSEQ; 419 420 if (!ret) 421 return 0; 422 423 vc->name = "meta"; 424 log_verify_failure(hdr, vc); 425 return ret; 426} 427 428static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc) 429{ 430 void *p = io_u_verify_off(hdr, vc); 431 struct vhdr_xxhash *vh = hdr_priv(hdr); 432 uint32_t hash; 433 void *state; 434 435 dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len); 436 437 state = XXH32_init(1); 438 XXH32_update(state, p, hdr->len - hdr_size(hdr)); 439 hash = XXH32_digest(state); 440 441 if (vh->hash == hash) 442 return 0; 443 444 vc->name = "xxhash"; 445 vc->good_crc = &vh->hash; 446 vc->bad_crc = &hash; 447 vc->crc_len = sizeof(hash); 448 log_verify_failure(hdr, vc); 449 return EILSEQ; 450} 451 452static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 453{ 454 void *p = io_u_verify_off(hdr, vc); 455 struct vhdr_sha512 *vh = hdr_priv(hdr); 456 uint8_t sha512[128]; 457 struct fio_sha512_ctx sha512_ctx = { 458 .buf = sha512, 459 }; 460 461 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 462 463 fio_sha512_init(&sha512_ctx); 464 fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 465 466 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 467 return 0; 468 469 vc->name = "sha512"; 470 vc->good_crc = vh->sha512; 471 vc->bad_crc = sha512_ctx.buf; 472 vc->crc_len = sizeof(vh->sha512); 473 log_verify_failure(hdr, vc); 474 return EILSEQ; 475} 476 477static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 478{ 479 void *p = io_u_verify_off(hdr, vc); 480 struct vhdr_sha256 *vh = hdr_priv(hdr); 481 uint8_t sha256[64]; 482 struct fio_sha256_ctx sha256_ctx = { 483 .buf = sha256, 484 }; 485 486 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 487 488 fio_sha256_init(&sha256_ctx); 489 fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 490 491 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 492 return 0; 493 494 vc->name = "sha256"; 495 vc->good_crc = vh->sha256; 496 vc->bad_crc = sha256_ctx.buf; 497 vc->crc_len = sizeof(vh->sha256); 498 log_verify_failure(hdr, vc); 499 return EILSEQ; 500} 501 502static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 503{ 504 void *p = io_u_verify_off(hdr, vc); 505 struct vhdr_sha1 *vh = hdr_priv(hdr); 506 uint32_t sha1[5]; 507 struct fio_sha1_ctx sha1_ctx = { 508 .H = sha1, 509 }; 510 511 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 512 513 fio_sha1_init(&sha1_ctx); 514 fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 515 516 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 517 return 0; 518 519 vc->name = "sha1"; 520 vc->good_crc = vh->sha1; 521 vc->bad_crc = sha1_ctx.H; 522 vc->crc_len = sizeof(vh->sha1); 523 log_verify_failure(hdr, vc); 524 return EILSEQ; 525} 526 527static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 528{ 529 void *p = io_u_verify_off(hdr, vc); 530 struct vhdr_crc7 *vh = hdr_priv(hdr); 531 unsigned char c; 532 533 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 534 535 c = fio_crc7(p, hdr->len - hdr_size(hdr)); 536 537 if (c == vh->crc7) 538 return 0; 539 540 vc->name = "crc7"; 541 vc->good_crc = &vh->crc7; 542 vc->bad_crc = &c; 543 vc->crc_len = 1; 544 log_verify_failure(hdr, vc); 545 return EILSEQ; 546} 547 548static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 549{ 550 void *p = io_u_verify_off(hdr, vc); 551 struct vhdr_crc16 *vh = hdr_priv(hdr); 552 unsigned short c; 553 554 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 555 556 c = fio_crc16(p, hdr->len - hdr_size(hdr)); 557 558 if (c == vh->crc16) 559 return 0; 560 561 vc->name = "crc16"; 562 vc->good_crc = &vh->crc16; 563 vc->bad_crc = &c; 564 vc->crc_len = 2; 565 log_verify_failure(hdr, vc); 566 return EILSEQ; 567} 568 569static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 570{ 571 void *p = io_u_verify_off(hdr, vc); 572 struct vhdr_crc64 *vh = hdr_priv(hdr); 573 unsigned long long c; 574 575 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 576 577 c = fio_crc64(p, hdr->len - hdr_size(hdr)); 578 579 if (c == vh->crc64) 580 return 0; 581 582 vc->name = "crc64"; 583 vc->good_crc = &vh->crc64; 584 vc->bad_crc = &c; 585 vc->crc_len = 8; 586 log_verify_failure(hdr, vc); 587 return EILSEQ; 588} 589 590static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 591{ 592 void *p = io_u_verify_off(hdr, vc); 593 struct vhdr_crc32 *vh = hdr_priv(hdr); 594 uint32_t c; 595 596 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 597 598 c = fio_crc32(p, hdr->len - hdr_size(hdr)); 599 600 if (c == vh->crc32) 601 return 0; 602 603 vc->name = "crc32"; 604 vc->good_crc = &vh->crc32; 605 vc->bad_crc = &c; 606 vc->crc_len = 4; 607 log_verify_failure(hdr, vc); 608 return EILSEQ; 609} 610 611static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 612{ 613 void *p = io_u_verify_off(hdr, vc); 614 struct vhdr_crc32 *vh = hdr_priv(hdr); 615 uint32_t c; 616 617 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 618 619 c = fio_crc32c(p, hdr->len - hdr_size(hdr)); 620 621 if (c == vh->crc32) 622 return 0; 623 624 vc->name = "crc32c"; 625 vc->good_crc = &vh->crc32; 626 vc->bad_crc = &c; 627 vc->crc_len = 4; 628 log_verify_failure(hdr, vc); 629 return EILSEQ; 630} 631 632static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 633{ 634 void *p = io_u_verify_off(hdr, vc); 635 struct vhdr_md5 *vh = hdr_priv(hdr); 636 uint32_t hash[MD5_HASH_WORDS]; 637 struct fio_md5_ctx md5_ctx = { 638 .hash = hash, 639 }; 640 641 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 642 643 fio_md5_init(&md5_ctx); 644 fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 645 646 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 647 return 0; 648 649 vc->name = "md5"; 650 vc->good_crc = vh->md5_digest; 651 vc->bad_crc = md5_ctx.hash; 652 vc->crc_len = sizeof(hash); 653 log_verify_failure(hdr, vc); 654 return EILSEQ; 655} 656 657/* 658 * Push IO verification to a separate thread 659 */ 660int verify_io_u_async(struct thread_data *td, struct io_u **io_u_ptr) 661{ 662 struct io_u *io_u = *io_u_ptr; 663 664 pthread_mutex_lock(&td->io_u_lock); 665 666 if (io_u->file) 667 put_file_log(td, io_u->file); 668 669 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 670 td->cur_depth--; 671 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 672 } 673 flist_add_tail(&io_u->verify_list, &td->verify_list); 674 *io_u_ptr = NULL; 675 pthread_mutex_unlock(&td->io_u_lock); 676 677 pthread_cond_signal(&td->verify_cond); 678 return 0; 679} 680 681static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 682{ 683 static char zero_buf[1024]; 684 unsigned int this_len, len; 685 int ret = 0; 686 void *p; 687 688 if (!td->o.trim_zero) 689 return 0; 690 691 len = io_u->buflen; 692 p = io_u->buf; 693 do { 694 this_len = sizeof(zero_buf); 695 if (this_len > len) 696 this_len = len; 697 if (memcmp(p, zero_buf, this_len)) { 698 ret = EILSEQ; 699 break; 700 } 701 len -= this_len; 702 p += this_len; 703 } while (len); 704 705 if (!ret) 706 return 0; 707 708 log_err("trim: verify failed at file %s offset %llu, length %lu" 709 ", block offset %lu\n", 710 io_u->file->file_name, io_u->offset, io_u->buflen, 711 (unsigned long) (p - io_u->buf)); 712 return ret; 713} 714 715static int verify_header(struct io_u *io_u, struct verify_header *hdr, 716 unsigned int hdr_num, unsigned int hdr_len) 717{ 718 void *p = hdr; 719 uint32_t crc; 720 721 if (hdr->magic != FIO_HDR_MAGIC) { 722 log_err("verify: bad magic header %x, wanted %x", 723 hdr->magic, FIO_HDR_MAGIC); 724 goto err; 725 } 726 if (hdr->len != hdr_len) { 727 log_err("verify: bad header length %u, wanted %u", 728 hdr->len, hdr_len); 729 goto err; 730 } 731 if (hdr->rand_seed != io_u->rand_seed) { 732 log_err("verify: bad header rand_seed %"PRIu64 733 ", wanted %"PRIu64, 734 hdr->rand_seed, io_u->rand_seed); 735 goto err; 736 } 737 738 crc = fio_crc32c(p, offsetof(struct verify_header, crc32)); 739 if (crc != hdr->crc32) { 740 log_err("verify: bad header crc %x, calculated %x", 741 hdr->crc32, crc); 742 goto err; 743 } 744 return 0; 745 746err: 747 log_err(" at file %s offset %llu, length %u\n", 748 io_u->file->file_name, 749 io_u->offset + hdr_num * hdr_len, hdr_len); 750 return EILSEQ; 751} 752 753int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr) 754{ 755 struct verify_header *hdr; 756 struct io_u *io_u = *io_u_ptr; 757 unsigned int header_size, hdr_inc, hdr_num = 0; 758 void *p; 759 int ret; 760 761 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 762 return 0; 763 /* 764 * If the IO engine is faking IO (like null), then just pretend 765 * we verified everything. 766 */ 767 if (td->io_ops->flags & FIO_FAKEIO) 768 return 0; 769 770 if (io_u->flags & IO_U_F_TRIMMED) { 771 ret = verify_trimmed_io_u(td, io_u); 772 goto done; 773 } 774 775 hdr_inc = get_hdr_inc(td, io_u); 776 777 ret = 0; 778 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 779 p += hdr_inc, hdr_num++) { 780 struct vcont vc = { 781 .io_u = io_u, 782 .hdr_num = hdr_num, 783 .td = td, 784 }; 785 unsigned int verify_type; 786 787 if (ret && td->o.verify_fatal) 788 break; 789 790 header_size = __hdr_size(td->o.verify); 791 if (td->o.verify_offset) 792 memswp(p, p + td->o.verify_offset, header_size); 793 hdr = p; 794 795 /* 796 * Make rand_seed check pass when have verifysort or 797 * verify_backlog. 798 */ 799 if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG)) 800 io_u->rand_seed = hdr->rand_seed; 801 802 ret = verify_header(io_u, hdr, hdr_num, hdr_inc); 803 if (ret) 804 return ret; 805 806 if (td->o.verify != VERIFY_NONE) 807 verify_type = td->o.verify; 808 else 809 verify_type = hdr->verify_type; 810 811 switch (verify_type) { 812 case VERIFY_MD5: 813 ret = verify_io_u_md5(hdr, &vc); 814 break; 815 case VERIFY_CRC64: 816 ret = verify_io_u_crc64(hdr, &vc); 817 break; 818 case VERIFY_CRC32C: 819 case VERIFY_CRC32C_INTEL: 820 ret = verify_io_u_crc32c(hdr, &vc); 821 break; 822 case VERIFY_CRC32: 823 ret = verify_io_u_crc32(hdr, &vc); 824 break; 825 case VERIFY_CRC16: 826 ret = verify_io_u_crc16(hdr, &vc); 827 break; 828 case VERIFY_CRC7: 829 ret = verify_io_u_crc7(hdr, &vc); 830 break; 831 case VERIFY_SHA256: 832 ret = verify_io_u_sha256(hdr, &vc); 833 break; 834 case VERIFY_SHA512: 835 ret = verify_io_u_sha512(hdr, &vc); 836 break; 837 case VERIFY_XXHASH: 838 ret = verify_io_u_xxhash(hdr, &vc); 839 break; 840 case VERIFY_META: 841 ret = verify_io_u_meta(hdr, &vc); 842 break; 843 case VERIFY_SHA1: 844 ret = verify_io_u_sha1(hdr, &vc); 845 break; 846 case VERIFY_PATTERN: 847 ret = verify_io_u_pattern(hdr, &vc); 848 break; 849 default: 850 log_err("Bad verify type %u\n", hdr->verify_type); 851 ret = EINVAL; 852 } 853 854 if (ret && verify_type != hdr->verify_type) 855 log_err("fio: verify type mismatch (%u media, %u given)\n", 856 hdr->verify_type, verify_type); 857 } 858 859done: 860 if (ret && td->o.verify_fatal) 861 fio_mark_td_terminate(td); 862 863 return ret; 864} 865 866static void fill_meta(struct verify_header *hdr, struct thread_data *td, 867 struct io_u *io_u, unsigned int header_num) 868{ 869 struct vhdr_meta *vh = hdr_priv(hdr); 870 871 vh->thread = td->thread_number; 872 873 vh->time_sec = io_u->start_time.tv_sec; 874 vh->time_usec = io_u->start_time.tv_usec; 875 876 vh->numberio = io_u->numberio; 877 878 vh->offset = io_u->offset + header_num * td->o.verify_interval; 879} 880 881static void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len) 882{ 883 struct vhdr_xxhash *vh = hdr_priv(hdr); 884 void *state; 885 886 state = XXH32_init(1); 887 XXH32_update(state, p, len); 888 vh->hash = XXH32_digest(state); 889} 890 891static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 892{ 893 struct vhdr_sha512 *vh = hdr_priv(hdr); 894 struct fio_sha512_ctx sha512_ctx = { 895 .buf = vh->sha512, 896 }; 897 898 fio_sha512_init(&sha512_ctx); 899 fio_sha512_update(&sha512_ctx, p, len); 900} 901 902static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 903{ 904 struct vhdr_sha256 *vh = hdr_priv(hdr); 905 struct fio_sha256_ctx sha256_ctx = { 906 .buf = vh->sha256, 907 }; 908 909 fio_sha256_init(&sha256_ctx); 910 fio_sha256_update(&sha256_ctx, p, len); 911} 912 913static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 914{ 915 struct vhdr_sha1 *vh = hdr_priv(hdr); 916 struct fio_sha1_ctx sha1_ctx = { 917 .H = vh->sha1, 918 }; 919 920 fio_sha1_init(&sha1_ctx); 921 fio_sha1_update(&sha1_ctx, p, len); 922} 923 924static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 925{ 926 struct vhdr_crc7 *vh = hdr_priv(hdr); 927 928 vh->crc7 = fio_crc7(p, len); 929} 930 931static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 932{ 933 struct vhdr_crc16 *vh = hdr_priv(hdr); 934 935 vh->crc16 = fio_crc16(p, len); 936} 937 938static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 939{ 940 struct vhdr_crc32 *vh = hdr_priv(hdr); 941 942 vh->crc32 = fio_crc32(p, len); 943} 944 945static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 946{ 947 struct vhdr_crc32 *vh = hdr_priv(hdr); 948 949 vh->crc32 = fio_crc32c(p, len); 950} 951 952static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 953{ 954 struct vhdr_crc64 *vh = hdr_priv(hdr); 955 956 vh->crc64 = fio_crc64(p, len); 957} 958 959static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 960{ 961 struct vhdr_md5 *vh = hdr_priv(hdr); 962 struct fio_md5_ctx md5_ctx = { 963 .hash = (uint32_t *) vh->md5_digest, 964 }; 965 966 fio_md5_init(&md5_ctx); 967 fio_md5_update(&md5_ctx, p, len); 968} 969 970static void populate_hdr(struct thread_data *td, struct io_u *io_u, 971 struct verify_header *hdr, unsigned int header_num, 972 unsigned int header_len) 973{ 974 unsigned int data_len; 975 void *data, *p; 976 977 p = (void *) hdr; 978 979 hdr->magic = FIO_HDR_MAGIC; 980 hdr->verify_type = td->o.verify; 981 hdr->len = header_len; 982 hdr->rand_seed = io_u->rand_seed; 983 hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); 984 985 data_len = header_len - hdr_size(hdr); 986 987 data = p + hdr_size(hdr); 988 switch (td->o.verify) { 989 case VERIFY_MD5: 990 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 991 io_u, hdr->len); 992 fill_md5(hdr, data, data_len); 993 break; 994 case VERIFY_CRC64: 995 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 996 io_u, hdr->len); 997 fill_crc64(hdr, data, data_len); 998 break; 999 case VERIFY_CRC32C: 1000 case VERIFY_CRC32C_INTEL: 1001 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 1002 io_u, hdr->len); 1003 fill_crc32c(hdr, data, data_len); 1004 break; 1005 case VERIFY_CRC32: 1006 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 1007 io_u, hdr->len); 1008 fill_crc32(hdr, data, data_len); 1009 break; 1010 case VERIFY_CRC16: 1011 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 1012 io_u, hdr->len); 1013 fill_crc16(hdr, data, data_len); 1014 break; 1015 case VERIFY_CRC7: 1016 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 1017 io_u, hdr->len); 1018 fill_crc7(hdr, data, data_len); 1019 break; 1020 case VERIFY_SHA256: 1021 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 1022 io_u, hdr->len); 1023 fill_sha256(hdr, data, data_len); 1024 break; 1025 case VERIFY_SHA512: 1026 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 1027 io_u, hdr->len); 1028 fill_sha512(hdr, data, data_len); 1029 break; 1030 case VERIFY_XXHASH: 1031 dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", 1032 io_u, hdr->len); 1033 fill_xxhash(hdr, data, data_len); 1034 break; 1035 case VERIFY_META: 1036 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 1037 io_u, hdr->len); 1038 fill_meta(hdr, td, io_u, header_num); 1039 break; 1040 case VERIFY_SHA1: 1041 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 1042 io_u, hdr->len); 1043 fill_sha1(hdr, data, data_len); 1044 break; 1045 case VERIFY_PATTERN: 1046 /* nothing to do here */ 1047 break; 1048 default: 1049 log_err("fio: bad verify type: %d\n", td->o.verify); 1050 assert(0); 1051 } 1052 if (td->o.verify_offset) 1053 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 1054} 1055 1056/* 1057 * fill body of io_u->buf with random data and add a header with the 1058 * checksum of choice 1059 */ 1060void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 1061{ 1062 if (td->o.verify == VERIFY_NULL) 1063 return; 1064 1065 io_u->numberio = td->io_issues[io_u->ddir]; 1066 1067 fill_pattern_headers(td, io_u, 0, 0); 1068} 1069 1070int get_next_verify(struct thread_data *td, struct io_u *io_u) 1071{ 1072 struct io_piece *ipo = NULL; 1073 1074 /* 1075 * this io_u is from a requeue, we already filled the offsets 1076 */ 1077 if (io_u->file) 1078 return 0; 1079 1080 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 1081 struct rb_node *n = rb_first(&td->io_hist_tree); 1082 1083 ipo = rb_entry(n, struct io_piece, rb_node); 1084 1085 /* 1086 * Ensure that the associated IO has completed 1087 */ 1088 read_barrier(); 1089 if (ipo->flags & IP_F_IN_FLIGHT) 1090 goto nothing; 1091 1092 rb_erase(n, &td->io_hist_tree); 1093 assert(ipo->flags & IP_F_ONRB); 1094 ipo->flags &= ~IP_F_ONRB; 1095 } else if (!flist_empty(&td->io_hist_list)) { 1096 ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list); 1097 1098 /* 1099 * Ensure that the associated IO has completed 1100 */ 1101 read_barrier(); 1102 if (ipo->flags & IP_F_IN_FLIGHT) 1103 goto nothing; 1104 1105 flist_del(&ipo->list); 1106 assert(ipo->flags & IP_F_ONLIST); 1107 ipo->flags &= ~IP_F_ONLIST; 1108 } 1109 1110 if (ipo) { 1111 td->io_hist_len--; 1112 1113 io_u->offset = ipo->offset; 1114 io_u->buflen = ipo->len; 1115 io_u->numberio = ipo->numberio; 1116 io_u->file = ipo->file; 1117 io_u->flags |= IO_U_F_VER_LIST; 1118 1119 if (ipo->flags & IP_F_TRIMMED) 1120 io_u->flags |= IO_U_F_TRIMMED; 1121 1122 if (!fio_file_open(io_u->file)) { 1123 int r = td_io_open_file(td, io_u->file); 1124 1125 if (r) { 1126 dprint(FD_VERIFY, "failed file %s open\n", 1127 io_u->file->file_name); 1128 return 1; 1129 } 1130 } 1131 1132 get_file(ipo->file); 1133 assert(fio_file_open(io_u->file)); 1134 io_u->ddir = DDIR_READ; 1135 io_u->xfer_buf = io_u->buf; 1136 io_u->xfer_buflen = io_u->buflen; 1137 1138 remove_trim_entry(td, ipo); 1139 free(ipo); 1140 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 1141 1142 if (!td->o.verify_pattern_bytes) { 1143 io_u->rand_seed = __rand(&td->verify_state); 1144 if (sizeof(int) != sizeof(long *)) 1145 io_u->rand_seed *= __rand(&td->verify_state); 1146 } 1147 return 0; 1148 } 1149 1150nothing: 1151 dprint(FD_VERIFY, "get_next_verify: empty\n"); 1152 return 1; 1153} 1154 1155void fio_verify_init(struct thread_data *td) 1156{ 1157 if (td->o.verify == VERIFY_CRC32C_INTEL || 1158 td->o.verify == VERIFY_CRC32C) { 1159 crc32c_intel_probe(); 1160 } 1161} 1162 1163static void *verify_async_thread(void *data) 1164{ 1165 struct thread_data *td = data; 1166 struct io_u *io_u; 1167 int ret = 0; 1168 1169 if (td->o.verify_cpumask_set && 1170 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 1171 log_err("fio: failed setting verify thread affinity\n"); 1172 goto done; 1173 } 1174 1175 do { 1176 FLIST_HEAD(list); 1177 1178 read_barrier(); 1179 if (td->verify_thread_exit) 1180 break; 1181 1182 pthread_mutex_lock(&td->io_u_lock); 1183 1184 while (flist_empty(&td->verify_list) && 1185 !td->verify_thread_exit) { 1186 ret = pthread_cond_wait(&td->verify_cond, 1187 &td->io_u_lock); 1188 if (ret) { 1189 pthread_mutex_unlock(&td->io_u_lock); 1190 break; 1191 } 1192 } 1193 1194 flist_splice_init(&td->verify_list, &list); 1195 pthread_mutex_unlock(&td->io_u_lock); 1196 1197 if (flist_empty(&list)) 1198 continue; 1199 1200 while (!flist_empty(&list)) { 1201 io_u = flist_first_entry(&list, struct io_u, verify_list); 1202 flist_del_init(&io_u->verify_list); 1203 1204 io_u->flags |= IO_U_F_NO_FILE_PUT; 1205 ret = verify_io_u(td, &io_u); 1206 1207 put_io_u(td, io_u); 1208 if (!ret) 1209 continue; 1210 if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { 1211 update_error_count(td, ret); 1212 td_clear_error(td); 1213 ret = 0; 1214 } 1215 } 1216 } while (!ret); 1217 1218 if (ret) { 1219 td_verror(td, ret, "async_verify"); 1220 if (td->o.verify_fatal) 1221 fio_mark_td_terminate(td); 1222 } 1223 1224done: 1225 pthread_mutex_lock(&td->io_u_lock); 1226 td->nr_verify_threads--; 1227 pthread_mutex_unlock(&td->io_u_lock); 1228 1229 pthread_cond_signal(&td->free_cond); 1230 return NULL; 1231} 1232 1233int verify_async_init(struct thread_data *td) 1234{ 1235 int i, ret; 1236 pthread_attr_t attr; 1237 1238 pthread_attr_init(&attr); 1239 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1240 1241 td->verify_thread_exit = 0; 1242 1243 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1244 for (i = 0; i < td->o.verify_async; i++) { 1245 ret = pthread_create(&td->verify_threads[i], &attr, 1246 verify_async_thread, td); 1247 if (ret) { 1248 log_err("fio: async verify creation failed: %s\n", 1249 strerror(ret)); 1250 break; 1251 } 1252 ret = pthread_detach(td->verify_threads[i]); 1253 if (ret) { 1254 log_err("fio: async verify thread detach failed: %s\n", 1255 strerror(ret)); 1256 break; 1257 } 1258 td->nr_verify_threads++; 1259 } 1260 1261 pthread_attr_destroy(&attr); 1262 1263 if (i != td->o.verify_async) { 1264 log_err("fio: only %d verify threads started, exiting\n", i); 1265 td->verify_thread_exit = 1; 1266 write_barrier(); 1267 pthread_cond_broadcast(&td->verify_cond); 1268 return 1; 1269 } 1270 1271 return 0; 1272} 1273 1274void verify_async_exit(struct thread_data *td) 1275{ 1276 td->verify_thread_exit = 1; 1277 write_barrier(); 1278 pthread_cond_broadcast(&td->verify_cond); 1279 1280 pthread_mutex_lock(&td->io_u_lock); 1281 1282 while (td->nr_verify_threads) 1283 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1284 1285 pthread_mutex_unlock(&td->io_u_lock); 1286 free(td->verify_threads); 1287 td->verify_threads = NULL; 1288} 1289 1290struct all_io_list *get_all_io_list(int save_mask, size_t *sz) 1291{ 1292 struct all_io_list *rep; 1293 struct thread_data *td; 1294 size_t depth; 1295 void *next; 1296 int i, nr; 1297 1298 compiletime_assert(sizeof(struct all_io_list) == 8, "all_io_list"); 1299 1300 /* 1301 * Calculate reply space needed. We need one 'io_state' per thread, 1302 * and the size will vary depending on depth. 1303 */ 1304 depth = 0; 1305 nr = 0; 1306 for_each_td(td, i) { 1307 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask) 1308 continue; 1309 td->stop_io = 1; 1310 td->flags |= TD_F_VSTATE_SAVED; 1311 depth += td->o.iodepth; 1312 nr++; 1313 } 1314 1315 if (!nr) 1316 return NULL; 1317 1318 *sz = sizeof(*rep); 1319 *sz += nr * sizeof(struct thread_io_list); 1320 *sz += depth * sizeof(uint64_t); 1321 rep = malloc(*sz); 1322 1323 rep->threads = cpu_to_le64((uint64_t) nr); 1324 1325 next = &rep->state[0]; 1326 for_each_td(td, i) { 1327 struct thread_io_list *s = next; 1328 unsigned int comps; 1329 1330 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask) 1331 continue; 1332 1333 if (td->last_write_comp) { 1334 int j, k; 1335 1336 if (td->io_blocks[DDIR_WRITE] < td->o.iodepth) 1337 comps = td->io_blocks[DDIR_WRITE]; 1338 else 1339 comps = td->o.iodepth; 1340 1341 k = td->last_write_idx - 1; 1342 for (j = 0; j < comps; j++) { 1343 if (k == -1) 1344 k = td->o.iodepth - 1; 1345 s->offsets[j] = cpu_to_le64(td->last_write_comp[k]); 1346 k--; 1347 } 1348 } else 1349 comps = 0; 1350 1351 s->no_comps = cpu_to_le64((uint64_t) comps); 1352 s->depth = cpu_to_le64((uint64_t) td->o.iodepth); 1353 s->numberio = cpu_to_le64((uint64_t) td->io_issues[DDIR_WRITE]); 1354 s->index = cpu_to_le64((uint64_t) i); 1355 s->rand.s[0] = cpu_to_le32(td->random_state.s1); 1356 s->rand.s[1] = cpu_to_le32(td->random_state.s2); 1357 s->rand.s[2] = cpu_to_le32(td->random_state.s3); 1358 s->rand.s[3] = 0; 1359 strncpy((char *) s->name, td->o.name, sizeof(s->name)); 1360 next = io_list_next(s); 1361 } 1362 1363 return rep; 1364} 1365 1366static int open_state_file(const char *name, const char *prefix, int num, 1367 int for_write) 1368{ 1369 char out[64]; 1370 int flags; 1371 int fd; 1372 1373 if (for_write) 1374 flags = O_CREAT | O_TRUNC | O_WRONLY | O_SYNC; 1375 else 1376 flags = O_RDONLY; 1377 1378 verify_state_gen_name(out, sizeof(out), name, prefix, num); 1379 1380 fd = open(out, flags, 0644); 1381 if (fd == -1) { 1382 perror("fio: open state file"); 1383 return -1; 1384 } 1385 1386 return fd; 1387} 1388 1389static int write_thread_list_state(struct thread_io_list *s, 1390 const char *prefix) 1391{ 1392 struct verify_state_hdr hdr; 1393 uint64_t crc; 1394 ssize_t ret; 1395 int fd; 1396 1397 fd = open_state_file((const char *) s->name, prefix, s->index, 1); 1398 if (fd == -1) 1399 return 1; 1400 1401 crc = fio_crc32c((void *)s, thread_io_list_sz(s)); 1402 1403 hdr.version = cpu_to_le64((uint64_t) VSTATE_HDR_VERSION); 1404 hdr.size = cpu_to_le64((uint64_t) thread_io_list_sz(s)); 1405 hdr.crc = cpu_to_le64(crc); 1406 ret = write(fd, &hdr, sizeof(hdr)); 1407 if (ret != sizeof(hdr)) 1408 goto write_fail; 1409 1410 ret = write(fd, s, thread_io_list_sz(s)); 1411 if (ret != thread_io_list_sz(s)) { 1412write_fail: 1413 if (ret < 0) 1414 perror("fio: write state file"); 1415 log_err("fio: failed to write state file\n"); 1416 ret = 1; 1417 } else 1418 ret = 0; 1419 1420 close(fd); 1421 return ret; 1422} 1423 1424void __verify_save_state(struct all_io_list *state, const char *prefix) 1425{ 1426 struct thread_io_list *s = &state->state[0]; 1427 unsigned int i; 1428 1429 for (i = 0; i < le64_to_cpu(state->threads); i++) { 1430 write_thread_list_state(s, prefix); 1431 s = io_list_next(s); 1432 } 1433} 1434 1435void verify_save_state(void) 1436{ 1437 struct all_io_list *state; 1438 size_t sz; 1439 1440 state = get_all_io_list(IO_LIST_ALL, &sz); 1441 if (state) { 1442 __verify_save_state(state, "local"); 1443 free(state); 1444 } 1445} 1446 1447void verify_free_state(struct thread_data *td) 1448{ 1449 if (td->vstate) 1450 free(td->vstate); 1451} 1452 1453void verify_convert_assign_state(struct thread_data *td, 1454 struct thread_io_list *s) 1455{ 1456 int i; 1457 1458 s->no_comps = le64_to_cpu(s->no_comps); 1459 s->depth = le64_to_cpu(s->depth); 1460 s->numberio = le64_to_cpu(s->numberio); 1461 for (i = 0; i < 4; i++) 1462 s->rand.s[i] = le32_to_cpu(s->rand.s[i]); 1463 for (i = 0; i < s->no_comps; i++) 1464 s->offsets[i] = le64_to_cpu(s->offsets[i]); 1465 1466 td->vstate = s; 1467} 1468 1469int verify_state_hdr(struct verify_state_hdr *hdr, struct thread_io_list *s) 1470{ 1471 uint64_t crc; 1472 1473 hdr->version = le64_to_cpu(hdr->version); 1474 hdr->size = le64_to_cpu(hdr->size); 1475 hdr->crc = le64_to_cpu(hdr->crc); 1476 1477 if (hdr->version != VSTATE_HDR_VERSION) 1478 return 1; 1479 1480 crc = fio_crc32c((void *)s, hdr->size); 1481 if (crc != hdr->crc) 1482 return 1; 1483 1484 return 0; 1485} 1486 1487int verify_load_state(struct thread_data *td, const char *prefix) 1488{ 1489 struct thread_io_list *s = NULL; 1490 struct verify_state_hdr hdr; 1491 uint64_t crc; 1492 ssize_t ret; 1493 int fd; 1494 1495 if (!td->o.verify_state) 1496 return 0; 1497 1498 fd = open_state_file(td->o.name, prefix, td->thread_number - 1, 0); 1499 if (fd == -1) 1500 return 1; 1501 1502 ret = read(fd, &hdr, sizeof(hdr)); 1503 if (ret != sizeof(hdr)) { 1504 if (ret < 0) 1505 td_verror(td, errno, "read verify state hdr"); 1506 log_err("fio: failed reading verify state header\n"); 1507 goto err; 1508 } 1509 1510 hdr.version = le64_to_cpu(hdr.version); 1511 hdr.size = le64_to_cpu(hdr.size); 1512 hdr.crc = le64_to_cpu(hdr.crc); 1513 1514 if (hdr.version != VSTATE_HDR_VERSION) { 1515 log_err("fio: bad version in verify state header\n"); 1516 goto err; 1517 } 1518 1519 s = malloc(hdr.size); 1520 ret = read(fd, s, hdr.size); 1521 if (ret != hdr.size) { 1522 if (ret < 0) 1523 td_verror(td, errno, "read verify state"); 1524 log_err("fio: failed reading verity state\n"); 1525 goto err; 1526 } 1527 1528 crc = fio_crc32c((void *)s, hdr.size); 1529 if (crc != hdr.crc) { 1530 log_err("fio: verify state is corrupt\n"); 1531 goto err; 1532 } 1533 1534 close(fd); 1535 1536 verify_convert_assign_state(td, s); 1537 return 0; 1538err: 1539 if (s) 1540 free(s); 1541 close(fd); 1542 return 1; 1543} 1544 1545/* 1546 * Use the loaded verify state to know when to stop doing verification 1547 */ 1548int verify_state_should_stop(struct thread_data *td, struct io_u *io_u) 1549{ 1550 struct thread_io_list *s = td->vstate; 1551 int i; 1552 1553 if (!s) 1554 return 0; 1555 1556 /* 1557 * If we're not into the window of issues - depth yet, continue 1558 */ 1559 if (td->io_blocks[DDIR_READ] < s->depth || 1560 s->numberio - td->io_blocks[DDIR_READ] > s->depth) 1561 return 0; 1562 1563 /* 1564 * We're in the window of having to check if this io was 1565 * completed or not. If the IO was seen as completed, then 1566 * lets verify it. 1567 */ 1568 for (i = 0; i < s->no_comps; i++) 1569 if (io_u->offset == s->offsets[i]) 1570 return 0; 1571 1572 /* 1573 * Not found, we have to stop 1574 */ 1575 return 1; 1576} 1577