verify.c revision d1a44b2fae5053c957cbde69144ba2ff342a7040
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9#include <libgen.h> 10 11#include "fio.h" 12#include "verify.h" 13#include "trim.h" 14#include "lib/rand.h" 15#include "lib/hweight.h" 16 17#include "crc/md5.h" 18#include "crc/crc64.h" 19#include "crc/crc32.h" 20#include "crc/crc32c.h" 21#include "crc/crc16.h" 22#include "crc/crc7.h" 23#include "crc/sha256.h" 24#include "crc/sha512.h" 25#include "crc/sha1.h" 26#include "crc/xxhash.h" 27 28static void populate_hdr(struct thread_data *td, struct io_u *io_u, 29 struct verify_header *hdr, unsigned int header_num, 30 unsigned int header_len); 31 32void fill_buffer_pattern(struct thread_data *td, void *p, unsigned int len) 33{ 34 fill_pattern(p, len, td->o.buffer_pattern, td->o.buffer_pattern_bytes); 35} 36 37void __fill_buffer(struct thread_options *o, unsigned long seed, void *p, 38 unsigned int len) 39{ 40 __fill_random_buf_percentage(seed, p, o->compress_percentage, len, len, o->buffer_pattern, o->buffer_pattern_bytes); 41} 42 43unsigned long fill_buffer(struct thread_data *td, void *p, unsigned int len) 44{ 45 struct frand_state *fs = &td->verify_state; 46 struct thread_options *o = &td->o; 47 48 return fill_random_buf_percentage(fs, p, o->compress_percentage, len, len, o->buffer_pattern, o->buffer_pattern_bytes); 49} 50 51void fill_verify_pattern(struct thread_data *td, void *p, unsigned int len, 52 struct io_u *io_u, unsigned long seed, int use_seed) 53{ 54 struct thread_options *o = &td->o; 55 56 if (!o->verify_pattern_bytes) { 57 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 58 59 if (use_seed) 60 __fill_buffer(o, seed, p, len); 61 else 62 io_u->rand_seed = fill_buffer(td, p, len); 63 return; 64 } 65 66 if (io_u->buf_filled_len >= len) { 67 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 68 o->verify_pattern_bytes, len); 69 return; 70 } 71 72 fill_pattern(p, len, o->verify_pattern, o->verify_pattern_bytes); 73 io_u->buf_filled_len = len; 74} 75 76static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u) 77{ 78 unsigned int hdr_inc; 79 80 hdr_inc = io_u->buflen; 81 if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen) 82 hdr_inc = td->o.verify_interval; 83 84 return hdr_inc; 85} 86 87static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 88 unsigned long seed, int use_seed) 89{ 90 unsigned int hdr_inc, header_num; 91 struct verify_header *hdr; 92 void *p = io_u->buf; 93 94 fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 95 96 hdr_inc = get_hdr_inc(td, io_u); 97 header_num = 0; 98 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 99 hdr = p; 100 populate_hdr(td, io_u, hdr, header_num, hdr_inc); 101 header_num++; 102 } 103} 104 105static void memswp(void *buf1, void *buf2, unsigned int len) 106{ 107 char swap[200]; 108 109 assert(len <= sizeof(swap)); 110 111 memcpy(&swap, buf1, len); 112 memcpy(buf1, buf2, len); 113 memcpy(buf2, &swap, len); 114} 115 116static void hexdump(void *buffer, int len) 117{ 118 unsigned char *p = buffer; 119 int i; 120 121 for (i = 0; i < len; i++) 122 log_err("%02x", p[i]); 123 log_err("\n"); 124} 125 126/* 127 * Prepare for separation of verify_header and checksum header 128 */ 129static inline unsigned int __hdr_size(int verify_type) 130{ 131 unsigned int len = 0; 132 133 switch (verify_type) { 134 case VERIFY_NONE: 135 case VERIFY_NULL: 136 len = 0; 137 break; 138 case VERIFY_MD5: 139 len = sizeof(struct vhdr_md5); 140 break; 141 case VERIFY_CRC64: 142 len = sizeof(struct vhdr_crc64); 143 break; 144 case VERIFY_CRC32C: 145 case VERIFY_CRC32: 146 case VERIFY_CRC32C_INTEL: 147 len = sizeof(struct vhdr_crc32); 148 break; 149 case VERIFY_CRC16: 150 len = sizeof(struct vhdr_crc16); 151 break; 152 case VERIFY_CRC7: 153 len = sizeof(struct vhdr_crc7); 154 break; 155 case VERIFY_SHA256: 156 len = sizeof(struct vhdr_sha256); 157 break; 158 case VERIFY_SHA512: 159 len = sizeof(struct vhdr_sha512); 160 break; 161 case VERIFY_XXHASH: 162 len = sizeof(struct vhdr_xxhash); 163 break; 164 case VERIFY_META: 165 len = sizeof(struct vhdr_meta); 166 break; 167 case VERIFY_SHA1: 168 len = sizeof(struct vhdr_sha1); 169 break; 170 case VERIFY_PATTERN: 171 len = 0; 172 break; 173 default: 174 log_err("fio: unknown verify header!\n"); 175 assert(0); 176 } 177 178 return len + sizeof(struct verify_header); 179} 180 181static inline unsigned int hdr_size(struct verify_header *hdr) 182{ 183 return __hdr_size(hdr->verify_type); 184} 185 186static void *hdr_priv(struct verify_header *hdr) 187{ 188 void *priv = hdr; 189 190 return priv + sizeof(struct verify_header); 191} 192 193/* 194 * Verify container, pass info to verify handlers and allow them to 195 * pass info back in case of error 196 */ 197struct vcont { 198 /* 199 * Input 200 */ 201 struct io_u *io_u; 202 unsigned int hdr_num; 203 struct thread_data *td; 204 205 /* 206 * Output, only valid in case of error 207 */ 208 const char *name; 209 void *good_crc; 210 void *bad_crc; 211 unsigned int crc_len; 212}; 213 214#define DUMP_BUF_SZ 255 215static int dump_buf_warned; 216 217static void dump_buf(char *buf, unsigned int len, unsigned long long offset, 218 const char *type, struct fio_file *f) 219{ 220 char *ptr, fname[DUMP_BUF_SZ]; 221 size_t buf_left = DUMP_BUF_SZ; 222 int ret, fd; 223 224 ptr = strdup(f->file_name); 225 226 fname[DUMP_BUF_SZ - 1] = '\0'; 227 strncpy(fname, basename(ptr), DUMP_BUF_SZ - 1); 228 229 buf_left -= strlen(fname); 230 if (buf_left <= 0) { 231 if (!dump_buf_warned) { 232 log_err("fio: verify failure dump buffer too small\n"); 233 dump_buf_warned = 1; 234 } 235 free(ptr); 236 return; 237 } 238 239 snprintf(fname + strlen(fname), buf_left, ".%llu.%s", offset, type); 240 241 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 242 if (fd < 0) { 243 perror("open verify buf file"); 244 return; 245 } 246 247 while (len) { 248 ret = write(fd, buf, len); 249 if (!ret) 250 break; 251 else if (ret < 0) { 252 perror("write verify buf file"); 253 break; 254 } 255 len -= ret; 256 buf += ret; 257 } 258 259 close(fd); 260 log_err(" %s data dumped as %s\n", type, fname); 261 free(ptr); 262} 263 264/* 265 * Dump the contents of the read block and re-generate the correct data 266 * and dump that too. 267 */ 268static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 269{ 270 struct thread_data *td = vc->td; 271 struct io_u *io_u = vc->io_u; 272 unsigned long hdr_offset; 273 struct io_u dummy; 274 void *buf; 275 276 if (!td->o.verify_dump) 277 return; 278 279 /* 280 * Dump the contents we just read off disk 281 */ 282 hdr_offset = vc->hdr_num * hdr->len; 283 284 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 285 "received", vc->io_u->file); 286 287 /* 288 * Allocate a new buf and re-generate the original data 289 */ 290 buf = malloc(io_u->buflen); 291 dummy = *io_u; 292 dummy.buf = buf; 293 dummy.rand_seed = hdr->rand_seed; 294 dummy.buf_filled_len = 0; 295 dummy.buflen = io_u->buflen; 296 297 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 298 299 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 300 "expected", vc->io_u->file); 301 free(buf); 302} 303 304static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 305{ 306 unsigned long long offset; 307 308 offset = vc->io_u->offset; 309 offset += vc->hdr_num * hdr->len; 310 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 311 vc->name, vc->io_u->file->file_name, offset, hdr->len); 312 313 if (vc->good_crc && vc->bad_crc) { 314 log_err(" Expected CRC: "); 315 hexdump(vc->good_crc, vc->crc_len); 316 log_err(" Received CRC: "); 317 hexdump(vc->bad_crc, vc->crc_len); 318 } 319 320 dump_verify_buffers(hdr, vc); 321} 322 323/* 324 * Return data area 'header_num' 325 */ 326static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 327{ 328 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 329} 330 331static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc) 332{ 333 struct thread_data *td = vc->td; 334 struct io_u *io_u = vc->io_u; 335 char *buf, *pattern; 336 unsigned int header_size = __hdr_size(td->o.verify); 337 unsigned int len, mod, i, size, pattern_size; 338 339 pattern = td->o.verify_pattern; 340 pattern_size = td->o.verify_pattern_bytes; 341 if (pattern_size <= 1) 342 pattern_size = MAX_PATTERN_SIZE; 343 buf = (void *) hdr + header_size; 344 len = get_hdr_inc(td, io_u) - header_size; 345 mod = header_size % pattern_size; 346 347 for (i = 0; i < len; i += size) { 348 size = pattern_size - mod; 349 if (size > (len - i)) 350 size = len - i; 351 if (memcmp(buf + i, pattern + mod, size)) 352 /* Let the slow compare find the first mismatch byte. */ 353 break; 354 mod = 0; 355 } 356 357 for (; i < len; i++) { 358 if (buf[i] != pattern[mod]) { 359 unsigned int bits; 360 361 bits = hweight8(buf[i] ^ pattern[mod]); 362 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 363 buf[i], pattern[mod], bits); 364 log_err("fio: bad pattern block offset %u\n", i); 365 dump_verify_buffers(hdr, vc); 366 return EILSEQ; 367 } 368 mod++; 369 if (mod == td->o.verify_pattern_bytes) 370 mod = 0; 371 } 372 373 return 0; 374} 375 376static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc) 377{ 378 struct thread_data *td = vc->td; 379 struct vhdr_meta *vh = hdr_priv(hdr); 380 struct io_u *io_u = vc->io_u; 381 int ret = EILSEQ; 382 383 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 384 385 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 386 ret = 0; 387 388 if (td->o.verify_pattern_bytes) 389 ret |= verify_io_u_pattern(hdr, vc); 390 391 /* 392 * For read-only workloads, the program cannot be certain of the 393 * last numberio written to a block. Checking of numberio will be 394 * done only for workloads that write data. For verify_only, 395 * numberio will be checked in the last iteration when the correct 396 * state of numberio, that would have been written to each block 397 * in a previous run of fio, has been reached. 398 */ 399 if ((td_write(td) || td_rw(td)) && (td_min_bs(td) == td_max_bs(td)) && 400 !td->o.time_based) 401 if (!td->o.verify_only || td->o.loops == 0) 402 if (vh->numberio != io_u->numberio) 403 ret = EILSEQ; 404 405 if (!ret) 406 return 0; 407 408 vc->name = "meta"; 409 log_verify_failure(hdr, vc); 410 return ret; 411} 412 413static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc) 414{ 415 void *p = io_u_verify_off(hdr, vc); 416 struct vhdr_xxhash *vh = hdr_priv(hdr); 417 uint32_t hash; 418 void *state; 419 420 dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len); 421 422 state = XXH32_init(1); 423 XXH32_update(state, p, hdr->len - hdr_size(hdr)); 424 hash = XXH32_digest(state); 425 426 if (vh->hash == hash) 427 return 0; 428 429 vc->name = "xxhash"; 430 vc->good_crc = &vh->hash; 431 vc->bad_crc = &hash; 432 vc->crc_len = sizeof(hash); 433 log_verify_failure(hdr, vc); 434 return EILSEQ; 435} 436 437static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 438{ 439 void *p = io_u_verify_off(hdr, vc); 440 struct vhdr_sha512 *vh = hdr_priv(hdr); 441 uint8_t sha512[128]; 442 struct fio_sha512_ctx sha512_ctx = { 443 .buf = sha512, 444 }; 445 446 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 447 448 fio_sha512_init(&sha512_ctx); 449 fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 450 451 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 452 return 0; 453 454 vc->name = "sha512"; 455 vc->good_crc = vh->sha512; 456 vc->bad_crc = sha512_ctx.buf; 457 vc->crc_len = sizeof(vh->sha512); 458 log_verify_failure(hdr, vc); 459 return EILSEQ; 460} 461 462static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 463{ 464 void *p = io_u_verify_off(hdr, vc); 465 struct vhdr_sha256 *vh = hdr_priv(hdr); 466 uint8_t sha256[64]; 467 struct fio_sha256_ctx sha256_ctx = { 468 .buf = sha256, 469 }; 470 471 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 472 473 fio_sha256_init(&sha256_ctx); 474 fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 475 476 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 477 return 0; 478 479 vc->name = "sha256"; 480 vc->good_crc = vh->sha256; 481 vc->bad_crc = sha256_ctx.buf; 482 vc->crc_len = sizeof(vh->sha256); 483 log_verify_failure(hdr, vc); 484 return EILSEQ; 485} 486 487static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 488{ 489 void *p = io_u_verify_off(hdr, vc); 490 struct vhdr_sha1 *vh = hdr_priv(hdr); 491 uint32_t sha1[5]; 492 struct fio_sha1_ctx sha1_ctx = { 493 .H = sha1, 494 }; 495 496 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 497 498 fio_sha1_init(&sha1_ctx); 499 fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 500 501 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 502 return 0; 503 504 vc->name = "sha1"; 505 vc->good_crc = vh->sha1; 506 vc->bad_crc = sha1_ctx.H; 507 vc->crc_len = sizeof(vh->sha1); 508 log_verify_failure(hdr, vc); 509 return EILSEQ; 510} 511 512static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 513{ 514 void *p = io_u_verify_off(hdr, vc); 515 struct vhdr_crc7 *vh = hdr_priv(hdr); 516 unsigned char c; 517 518 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 519 520 c = fio_crc7(p, hdr->len - hdr_size(hdr)); 521 522 if (c == vh->crc7) 523 return 0; 524 525 vc->name = "crc7"; 526 vc->good_crc = &vh->crc7; 527 vc->bad_crc = &c; 528 vc->crc_len = 1; 529 log_verify_failure(hdr, vc); 530 return EILSEQ; 531} 532 533static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 534{ 535 void *p = io_u_verify_off(hdr, vc); 536 struct vhdr_crc16 *vh = hdr_priv(hdr); 537 unsigned short c; 538 539 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 540 541 c = fio_crc16(p, hdr->len - hdr_size(hdr)); 542 543 if (c == vh->crc16) 544 return 0; 545 546 vc->name = "crc16"; 547 vc->good_crc = &vh->crc16; 548 vc->bad_crc = &c; 549 vc->crc_len = 2; 550 log_verify_failure(hdr, vc); 551 return EILSEQ; 552} 553 554static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 555{ 556 void *p = io_u_verify_off(hdr, vc); 557 struct vhdr_crc64 *vh = hdr_priv(hdr); 558 unsigned long long c; 559 560 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 561 562 c = fio_crc64(p, hdr->len - hdr_size(hdr)); 563 564 if (c == vh->crc64) 565 return 0; 566 567 vc->name = "crc64"; 568 vc->good_crc = &vh->crc64; 569 vc->bad_crc = &c; 570 vc->crc_len = 8; 571 log_verify_failure(hdr, vc); 572 return EILSEQ; 573} 574 575static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 576{ 577 void *p = io_u_verify_off(hdr, vc); 578 struct vhdr_crc32 *vh = hdr_priv(hdr); 579 uint32_t c; 580 581 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 582 583 c = fio_crc32(p, hdr->len - hdr_size(hdr)); 584 585 if (c == vh->crc32) 586 return 0; 587 588 vc->name = "crc32"; 589 vc->good_crc = &vh->crc32; 590 vc->bad_crc = &c; 591 vc->crc_len = 4; 592 log_verify_failure(hdr, vc); 593 return EILSEQ; 594} 595 596static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 597{ 598 void *p = io_u_verify_off(hdr, vc); 599 struct vhdr_crc32 *vh = hdr_priv(hdr); 600 uint32_t c; 601 602 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 603 604 c = fio_crc32c(p, hdr->len - hdr_size(hdr)); 605 606 if (c == vh->crc32) 607 return 0; 608 609 vc->name = "crc32c"; 610 vc->good_crc = &vh->crc32; 611 vc->bad_crc = &c; 612 vc->crc_len = 4; 613 log_verify_failure(hdr, vc); 614 return EILSEQ; 615} 616 617static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 618{ 619 void *p = io_u_verify_off(hdr, vc); 620 struct vhdr_md5 *vh = hdr_priv(hdr); 621 uint32_t hash[MD5_HASH_WORDS]; 622 struct fio_md5_ctx md5_ctx = { 623 .hash = hash, 624 }; 625 626 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 627 628 fio_md5_init(&md5_ctx); 629 fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 630 631 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 632 return 0; 633 634 vc->name = "md5"; 635 vc->good_crc = vh->md5_digest; 636 vc->bad_crc = md5_ctx.hash; 637 vc->crc_len = sizeof(hash); 638 log_verify_failure(hdr, vc); 639 return EILSEQ; 640} 641 642/* 643 * Push IO verification to a separate thread 644 */ 645int verify_io_u_async(struct thread_data *td, struct io_u **io_u_ptr) 646{ 647 struct io_u *io_u = *io_u_ptr; 648 649 pthread_mutex_lock(&td->io_u_lock); 650 651 if (io_u->file) 652 put_file_log(td, io_u->file); 653 654 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 655 td->cur_depth--; 656 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 657 } 658 flist_add_tail(&io_u->verify_list, &td->verify_list); 659 *io_u_ptr = NULL; 660 pthread_mutex_unlock(&td->io_u_lock); 661 662 pthread_cond_signal(&td->verify_cond); 663 return 0; 664} 665 666static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 667{ 668 static char zero_buf[1024]; 669 unsigned int this_len, len; 670 int ret = 0; 671 void *p; 672 673 if (!td->o.trim_zero) 674 return 0; 675 676 len = io_u->buflen; 677 p = io_u->buf; 678 do { 679 this_len = sizeof(zero_buf); 680 if (this_len > len) 681 this_len = len; 682 if (memcmp(p, zero_buf, this_len)) { 683 ret = EILSEQ; 684 break; 685 } 686 len -= this_len; 687 p += this_len; 688 } while (len); 689 690 if (!ret) 691 return 0; 692 693 log_err("trim: verify failed at file %s offset %llu, length %lu" 694 ", block offset %lu\n", 695 io_u->file->file_name, io_u->offset, io_u->buflen, 696 (unsigned long) (p - io_u->buf)); 697 return ret; 698} 699 700static int verify_header(struct io_u *io_u, struct verify_header *hdr, 701 unsigned int hdr_num, unsigned int hdr_len) 702{ 703 void *p = hdr; 704 uint32_t crc; 705 706 if (hdr->magic != FIO_HDR_MAGIC) { 707 log_err("verify: bad magic header %x, wanted %x", 708 hdr->magic, FIO_HDR_MAGIC); 709 goto err; 710 } 711 if (hdr->len != hdr_len) { 712 log_err("verify: bad header length %u, wanted %u", 713 hdr->len, hdr_len); 714 goto err; 715 } 716 if (hdr->rand_seed != io_u->rand_seed) { 717 log_err("verify: bad header rand_seed %"PRIu64 718 ", wanted %"PRIu64, 719 hdr->rand_seed, io_u->rand_seed); 720 goto err; 721 } 722 723 crc = fio_crc32c(p, offsetof(struct verify_header, crc32)); 724 if (crc != hdr->crc32) { 725 log_err("verify: bad header crc %x, calculated %x", 726 hdr->crc32, crc); 727 goto err; 728 } 729 return 0; 730 731err: 732 log_err(" at file %s offset %llu, length %u\n", 733 io_u->file->file_name, 734 io_u->offset + hdr_num * hdr_len, hdr_len); 735 return EILSEQ; 736} 737 738int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr) 739{ 740 struct verify_header *hdr; 741 struct io_u *io_u = *io_u_ptr; 742 unsigned int header_size, hdr_inc, hdr_num = 0; 743 void *p; 744 int ret; 745 746 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 747 return 0; 748 /* 749 * If the IO engine is faking IO (like null), then just pretend 750 * we verified everything. 751 */ 752 if (td->io_ops->flags & FIO_FAKEIO) 753 return 0; 754 755 if (io_u->flags & IO_U_F_TRIMMED) { 756 ret = verify_trimmed_io_u(td, io_u); 757 goto done; 758 } 759 760 hdr_inc = get_hdr_inc(td, io_u); 761 762 ret = 0; 763 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 764 p += hdr_inc, hdr_num++) { 765 struct vcont vc = { 766 .io_u = io_u, 767 .hdr_num = hdr_num, 768 .td = td, 769 }; 770 unsigned int verify_type; 771 772 if (ret && td->o.verify_fatal) 773 break; 774 775 header_size = __hdr_size(td->o.verify); 776 if (td->o.verify_offset) 777 memswp(p, p + td->o.verify_offset, header_size); 778 hdr = p; 779 780 /* 781 * Make rand_seed check pass when have verifysort or 782 * verify_backlog. 783 */ 784 if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG)) 785 io_u->rand_seed = hdr->rand_seed; 786 787 ret = verify_header(io_u, hdr, hdr_num, hdr_inc); 788 if (ret) 789 return ret; 790 791 if (td->o.verify != VERIFY_NONE) 792 verify_type = td->o.verify; 793 else 794 verify_type = hdr->verify_type; 795 796 switch (verify_type) { 797 case VERIFY_MD5: 798 ret = verify_io_u_md5(hdr, &vc); 799 break; 800 case VERIFY_CRC64: 801 ret = verify_io_u_crc64(hdr, &vc); 802 break; 803 case VERIFY_CRC32C: 804 case VERIFY_CRC32C_INTEL: 805 ret = verify_io_u_crc32c(hdr, &vc); 806 break; 807 case VERIFY_CRC32: 808 ret = verify_io_u_crc32(hdr, &vc); 809 break; 810 case VERIFY_CRC16: 811 ret = verify_io_u_crc16(hdr, &vc); 812 break; 813 case VERIFY_CRC7: 814 ret = verify_io_u_crc7(hdr, &vc); 815 break; 816 case VERIFY_SHA256: 817 ret = verify_io_u_sha256(hdr, &vc); 818 break; 819 case VERIFY_SHA512: 820 ret = verify_io_u_sha512(hdr, &vc); 821 break; 822 case VERIFY_XXHASH: 823 ret = verify_io_u_xxhash(hdr, &vc); 824 break; 825 case VERIFY_META: 826 ret = verify_io_u_meta(hdr, &vc); 827 break; 828 case VERIFY_SHA1: 829 ret = verify_io_u_sha1(hdr, &vc); 830 break; 831 case VERIFY_PATTERN: 832 ret = verify_io_u_pattern(hdr, &vc); 833 break; 834 default: 835 log_err("Bad verify type %u\n", hdr->verify_type); 836 ret = EINVAL; 837 } 838 839 if (ret && verify_type != hdr->verify_type) 840 log_err("fio: verify type mismatch (%u media, %u given)\n", 841 hdr->verify_type, verify_type); 842 } 843 844done: 845 if (ret && td->o.verify_fatal) 846 fio_mark_td_terminate(td); 847 848 return ret; 849} 850 851static void fill_meta(struct verify_header *hdr, struct thread_data *td, 852 struct io_u *io_u, unsigned int header_num) 853{ 854 struct vhdr_meta *vh = hdr_priv(hdr); 855 856 vh->thread = td->thread_number; 857 858 vh->time_sec = io_u->start_time.tv_sec; 859 vh->time_usec = io_u->start_time.tv_usec; 860 861 vh->numberio = io_u->numberio; 862 863 vh->offset = io_u->offset + header_num * td->o.verify_interval; 864} 865 866static void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len) 867{ 868 struct vhdr_xxhash *vh = hdr_priv(hdr); 869 void *state; 870 871 state = XXH32_init(1); 872 XXH32_update(state, p, len); 873 vh->hash = XXH32_digest(state); 874} 875 876static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 877{ 878 struct vhdr_sha512 *vh = hdr_priv(hdr); 879 struct fio_sha512_ctx sha512_ctx = { 880 .buf = vh->sha512, 881 }; 882 883 fio_sha512_init(&sha512_ctx); 884 fio_sha512_update(&sha512_ctx, p, len); 885} 886 887static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 888{ 889 struct vhdr_sha256 *vh = hdr_priv(hdr); 890 struct fio_sha256_ctx sha256_ctx = { 891 .buf = vh->sha256, 892 }; 893 894 fio_sha256_init(&sha256_ctx); 895 fio_sha256_update(&sha256_ctx, p, len); 896} 897 898static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 899{ 900 struct vhdr_sha1 *vh = hdr_priv(hdr); 901 struct fio_sha1_ctx sha1_ctx = { 902 .H = vh->sha1, 903 }; 904 905 fio_sha1_init(&sha1_ctx); 906 fio_sha1_update(&sha1_ctx, p, len); 907} 908 909static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 910{ 911 struct vhdr_crc7 *vh = hdr_priv(hdr); 912 913 vh->crc7 = fio_crc7(p, len); 914} 915 916static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 917{ 918 struct vhdr_crc16 *vh = hdr_priv(hdr); 919 920 vh->crc16 = fio_crc16(p, len); 921} 922 923static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 924{ 925 struct vhdr_crc32 *vh = hdr_priv(hdr); 926 927 vh->crc32 = fio_crc32(p, len); 928} 929 930static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 931{ 932 struct vhdr_crc32 *vh = hdr_priv(hdr); 933 934 vh->crc32 = fio_crc32c(p, len); 935} 936 937static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 938{ 939 struct vhdr_crc64 *vh = hdr_priv(hdr); 940 941 vh->crc64 = fio_crc64(p, len); 942} 943 944static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 945{ 946 struct vhdr_md5 *vh = hdr_priv(hdr); 947 struct fio_md5_ctx md5_ctx = { 948 .hash = (uint32_t *) vh->md5_digest, 949 }; 950 951 fio_md5_init(&md5_ctx); 952 fio_md5_update(&md5_ctx, p, len); 953} 954 955static void populate_hdr(struct thread_data *td, struct io_u *io_u, 956 struct verify_header *hdr, unsigned int header_num, 957 unsigned int header_len) 958{ 959 unsigned int data_len; 960 void *data, *p; 961 962 p = (void *) hdr; 963 964 hdr->magic = FIO_HDR_MAGIC; 965 hdr->verify_type = td->o.verify; 966 hdr->len = header_len; 967 hdr->rand_seed = io_u->rand_seed; 968 hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); 969 970 data_len = header_len - hdr_size(hdr); 971 972 data = p + hdr_size(hdr); 973 switch (td->o.verify) { 974 case VERIFY_MD5: 975 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 976 io_u, hdr->len); 977 fill_md5(hdr, data, data_len); 978 break; 979 case VERIFY_CRC64: 980 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 981 io_u, hdr->len); 982 fill_crc64(hdr, data, data_len); 983 break; 984 case VERIFY_CRC32C: 985 case VERIFY_CRC32C_INTEL: 986 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 987 io_u, hdr->len); 988 fill_crc32c(hdr, data, data_len); 989 break; 990 case VERIFY_CRC32: 991 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 992 io_u, hdr->len); 993 fill_crc32(hdr, data, data_len); 994 break; 995 case VERIFY_CRC16: 996 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 997 io_u, hdr->len); 998 fill_crc16(hdr, data, data_len); 999 break; 1000 case VERIFY_CRC7: 1001 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 1002 io_u, hdr->len); 1003 fill_crc7(hdr, data, data_len); 1004 break; 1005 case VERIFY_SHA256: 1006 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 1007 io_u, hdr->len); 1008 fill_sha256(hdr, data, data_len); 1009 break; 1010 case VERIFY_SHA512: 1011 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 1012 io_u, hdr->len); 1013 fill_sha512(hdr, data, data_len); 1014 break; 1015 case VERIFY_XXHASH: 1016 dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", 1017 io_u, hdr->len); 1018 fill_xxhash(hdr, data, data_len); 1019 break; 1020 case VERIFY_META: 1021 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 1022 io_u, hdr->len); 1023 fill_meta(hdr, td, io_u, header_num); 1024 break; 1025 case VERIFY_SHA1: 1026 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 1027 io_u, hdr->len); 1028 fill_sha1(hdr, data, data_len); 1029 break; 1030 case VERIFY_PATTERN: 1031 /* nothing to do here */ 1032 break; 1033 default: 1034 log_err("fio: bad verify type: %d\n", td->o.verify); 1035 assert(0); 1036 } 1037 if (td->o.verify_offset) 1038 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 1039} 1040 1041/* 1042 * fill body of io_u->buf with random data and add a header with the 1043 * checksum of choice 1044 */ 1045void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 1046{ 1047 if (td->o.verify == VERIFY_NULL) 1048 return; 1049 1050 io_u->numberio = td->io_issues[io_u->ddir]; 1051 1052 fill_pattern_headers(td, io_u, 0, 0); 1053} 1054 1055int get_next_verify(struct thread_data *td, struct io_u *io_u) 1056{ 1057 struct io_piece *ipo = NULL; 1058 1059 /* 1060 * this io_u is from a requeue, we already filled the offsets 1061 */ 1062 if (io_u->file) 1063 return 0; 1064 1065 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 1066 struct rb_node *n = rb_first(&td->io_hist_tree); 1067 1068 ipo = rb_entry(n, struct io_piece, rb_node); 1069 1070 /* 1071 * Ensure that the associated IO has completed 1072 */ 1073 read_barrier(); 1074 if (ipo->flags & IP_F_IN_FLIGHT) 1075 goto nothing; 1076 1077 rb_erase(n, &td->io_hist_tree); 1078 assert(ipo->flags & IP_F_ONRB); 1079 ipo->flags &= ~IP_F_ONRB; 1080 } else if (!flist_empty(&td->io_hist_list)) { 1081 ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list); 1082 1083 /* 1084 * Ensure that the associated IO has completed 1085 */ 1086 read_barrier(); 1087 if (ipo->flags & IP_F_IN_FLIGHT) 1088 goto nothing; 1089 1090 flist_del(&ipo->list); 1091 assert(ipo->flags & IP_F_ONLIST); 1092 ipo->flags &= ~IP_F_ONLIST; 1093 } 1094 1095 if (ipo) { 1096 td->io_hist_len--; 1097 1098 io_u->offset = ipo->offset; 1099 io_u->buflen = ipo->len; 1100 io_u->numberio = ipo->numberio; 1101 io_u->file = ipo->file; 1102 io_u->flags |= IO_U_F_VER_LIST; 1103 1104 if (ipo->flags & IP_F_TRIMMED) 1105 io_u->flags |= IO_U_F_TRIMMED; 1106 1107 if (!fio_file_open(io_u->file)) { 1108 int r = td_io_open_file(td, io_u->file); 1109 1110 if (r) { 1111 dprint(FD_VERIFY, "failed file %s open\n", 1112 io_u->file->file_name); 1113 return 1; 1114 } 1115 } 1116 1117 get_file(ipo->file); 1118 assert(fio_file_open(io_u->file)); 1119 io_u->ddir = DDIR_READ; 1120 io_u->xfer_buf = io_u->buf; 1121 io_u->xfer_buflen = io_u->buflen; 1122 1123 remove_trim_entry(td, ipo); 1124 free(ipo); 1125 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 1126 1127 if (!td->o.verify_pattern_bytes) { 1128 io_u->rand_seed = __rand(&td->verify_state); 1129 if (sizeof(int) != sizeof(long *)) 1130 io_u->rand_seed *= __rand(&td->verify_state); 1131 } 1132 return 0; 1133 } 1134 1135nothing: 1136 dprint(FD_VERIFY, "get_next_verify: empty\n"); 1137 return 1; 1138} 1139 1140void fio_verify_init(struct thread_data *td) 1141{ 1142 if (td->o.verify == VERIFY_CRC32C_INTEL || 1143 td->o.verify == VERIFY_CRC32C) { 1144 crc32c_intel_probe(); 1145 } 1146} 1147 1148static void *verify_async_thread(void *data) 1149{ 1150 struct thread_data *td = data; 1151 struct io_u *io_u; 1152 int ret = 0; 1153 1154 if (fio_option_is_set(&td->o, verify_cpumask) && 1155 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 1156 log_err("fio: failed setting verify thread affinity\n"); 1157 goto done; 1158 } 1159 1160 do { 1161 FLIST_HEAD(list); 1162 1163 read_barrier(); 1164 if (td->verify_thread_exit) 1165 break; 1166 1167 pthread_mutex_lock(&td->io_u_lock); 1168 1169 while (flist_empty(&td->verify_list) && 1170 !td->verify_thread_exit) { 1171 ret = pthread_cond_wait(&td->verify_cond, 1172 &td->io_u_lock); 1173 if (ret) { 1174 pthread_mutex_unlock(&td->io_u_lock); 1175 break; 1176 } 1177 } 1178 1179 flist_splice_init(&td->verify_list, &list); 1180 pthread_mutex_unlock(&td->io_u_lock); 1181 1182 if (flist_empty(&list)) 1183 continue; 1184 1185 while (!flist_empty(&list)) { 1186 io_u = flist_first_entry(&list, struct io_u, verify_list); 1187 flist_del_init(&io_u->verify_list); 1188 1189 io_u->flags |= IO_U_F_NO_FILE_PUT; 1190 ret = verify_io_u(td, &io_u); 1191 1192 put_io_u(td, io_u); 1193 if (!ret) 1194 continue; 1195 if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { 1196 update_error_count(td, ret); 1197 td_clear_error(td); 1198 ret = 0; 1199 } 1200 } 1201 } while (!ret); 1202 1203 if (ret) { 1204 td_verror(td, ret, "async_verify"); 1205 if (td->o.verify_fatal) 1206 fio_mark_td_terminate(td); 1207 } 1208 1209done: 1210 pthread_mutex_lock(&td->io_u_lock); 1211 td->nr_verify_threads--; 1212 pthread_mutex_unlock(&td->io_u_lock); 1213 1214 pthread_cond_signal(&td->free_cond); 1215 return NULL; 1216} 1217 1218int verify_async_init(struct thread_data *td) 1219{ 1220 int i, ret; 1221 pthread_attr_t attr; 1222 1223 pthread_attr_init(&attr); 1224 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1225 1226 td->verify_thread_exit = 0; 1227 1228 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1229 for (i = 0; i < td->o.verify_async; i++) { 1230 ret = pthread_create(&td->verify_threads[i], &attr, 1231 verify_async_thread, td); 1232 if (ret) { 1233 log_err("fio: async verify creation failed: %s\n", 1234 strerror(ret)); 1235 break; 1236 } 1237 ret = pthread_detach(td->verify_threads[i]); 1238 if (ret) { 1239 log_err("fio: async verify thread detach failed: %s\n", 1240 strerror(ret)); 1241 break; 1242 } 1243 td->nr_verify_threads++; 1244 } 1245 1246 pthread_attr_destroy(&attr); 1247 1248 if (i != td->o.verify_async) { 1249 log_err("fio: only %d verify threads started, exiting\n", i); 1250 td->verify_thread_exit = 1; 1251 write_barrier(); 1252 pthread_cond_broadcast(&td->verify_cond); 1253 return 1; 1254 } 1255 1256 return 0; 1257} 1258 1259void verify_async_exit(struct thread_data *td) 1260{ 1261 td->verify_thread_exit = 1; 1262 write_barrier(); 1263 pthread_cond_broadcast(&td->verify_cond); 1264 1265 pthread_mutex_lock(&td->io_u_lock); 1266 1267 while (td->nr_verify_threads) 1268 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1269 1270 pthread_mutex_unlock(&td->io_u_lock); 1271 free(td->verify_threads); 1272 td->verify_threads = NULL; 1273} 1274 1275struct all_io_list *get_all_io_list(int save_mask, size_t *sz) 1276{ 1277 struct all_io_list *rep; 1278 struct thread_data *td; 1279 size_t depth; 1280 void *next; 1281 int i, nr; 1282 1283 compiletime_assert(sizeof(struct all_io_list) == 8, "all_io_list"); 1284 1285 /* 1286 * Calculate reply space needed. We need one 'io_state' per thread, 1287 * and the size will vary depending on depth. 1288 */ 1289 depth = 0; 1290 nr = 0; 1291 for_each_td(td, i) { 1292 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask) 1293 continue; 1294 td->stop_io = 1; 1295 td->flags |= TD_F_VSTATE_SAVED; 1296 depth += td->o.iodepth; 1297 nr++; 1298 } 1299 1300 if (!nr) 1301 return NULL; 1302 1303 *sz = sizeof(*rep); 1304 *sz += nr * sizeof(struct thread_io_list); 1305 *sz += depth * sizeof(uint64_t); 1306 rep = malloc(*sz); 1307 1308 rep->threads = cpu_to_le64((uint64_t) nr); 1309 1310 next = &rep->state[0]; 1311 for_each_td(td, i) { 1312 struct thread_io_list *s = next; 1313 unsigned int comps; 1314 1315 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask) 1316 continue; 1317 1318 if (td->last_write_comp) { 1319 int j, k; 1320 1321 if (td->io_blocks[DDIR_WRITE] < td->o.iodepth) 1322 comps = td->io_blocks[DDIR_WRITE]; 1323 else 1324 comps = td->o.iodepth; 1325 1326 k = td->last_write_idx - 1; 1327 for (j = 0; j < comps; j++) { 1328 if (k == -1) 1329 k = td->o.iodepth - 1; 1330 s->offsets[j] = cpu_to_le64(td->last_write_comp[k]); 1331 k--; 1332 } 1333 } else 1334 comps = 0; 1335 1336 s->no_comps = cpu_to_le64((uint64_t) comps); 1337 s->depth = cpu_to_le64((uint64_t) td->o.iodepth); 1338 s->numberio = cpu_to_le64((uint64_t) td->io_issues[DDIR_WRITE]); 1339 s->index = cpu_to_le64((uint64_t) i); 1340 s->rand.s[0] = cpu_to_le32(td->random_state.s1); 1341 s->rand.s[1] = cpu_to_le32(td->random_state.s2); 1342 s->rand.s[2] = cpu_to_le32(td->random_state.s3); 1343 s->rand.s[3] = 0; 1344 s->name[sizeof(s->name) - 1] = '\0'; 1345 strncpy((char *) s->name, td->o.name, sizeof(s->name) - 1); 1346 next = io_list_next(s); 1347 } 1348 1349 return rep; 1350} 1351 1352static int open_state_file(const char *name, const char *prefix, int num, 1353 int for_write) 1354{ 1355 char out[64]; 1356 int flags; 1357 int fd; 1358 1359 if (for_write) 1360 flags = O_CREAT | O_TRUNC | O_WRONLY | O_SYNC; 1361 else 1362 flags = O_RDONLY; 1363 1364 verify_state_gen_name(out, sizeof(out), name, prefix, num); 1365 1366 fd = open(out, flags, 0644); 1367 if (fd == -1) { 1368 perror("fio: open state file"); 1369 return -1; 1370 } 1371 1372 return fd; 1373} 1374 1375static int write_thread_list_state(struct thread_io_list *s, 1376 const char *prefix) 1377{ 1378 struct verify_state_hdr hdr; 1379 uint64_t crc; 1380 ssize_t ret; 1381 int fd; 1382 1383 fd = open_state_file((const char *) s->name, prefix, s->index, 1); 1384 if (fd == -1) 1385 return 1; 1386 1387 crc = fio_crc32c((void *)s, thread_io_list_sz(s)); 1388 1389 hdr.version = cpu_to_le64((uint64_t) VSTATE_HDR_VERSION); 1390 hdr.size = cpu_to_le64((uint64_t) thread_io_list_sz(s)); 1391 hdr.crc = cpu_to_le64(crc); 1392 ret = write(fd, &hdr, sizeof(hdr)); 1393 if (ret != sizeof(hdr)) 1394 goto write_fail; 1395 1396 ret = write(fd, s, thread_io_list_sz(s)); 1397 if (ret != thread_io_list_sz(s)) { 1398write_fail: 1399 if (ret < 0) 1400 perror("fio: write state file"); 1401 log_err("fio: failed to write state file\n"); 1402 ret = 1; 1403 } else 1404 ret = 0; 1405 1406 close(fd); 1407 return ret; 1408} 1409 1410void __verify_save_state(struct all_io_list *state, const char *prefix) 1411{ 1412 struct thread_io_list *s = &state->state[0]; 1413 unsigned int i; 1414 1415 for (i = 0; i < le64_to_cpu(state->threads); i++) { 1416 write_thread_list_state(s, prefix); 1417 s = io_list_next(s); 1418 } 1419} 1420 1421void verify_save_state(void) 1422{ 1423 struct all_io_list *state; 1424 size_t sz; 1425 1426 state = get_all_io_list(IO_LIST_ALL, &sz); 1427 if (state) { 1428 __verify_save_state(state, "local"); 1429 free(state); 1430 } 1431} 1432 1433void verify_free_state(struct thread_data *td) 1434{ 1435 if (td->vstate) 1436 free(td->vstate); 1437} 1438 1439void verify_convert_assign_state(struct thread_data *td, 1440 struct thread_io_list *s) 1441{ 1442 int i; 1443 1444 s->no_comps = le64_to_cpu(s->no_comps); 1445 s->depth = le64_to_cpu(s->depth); 1446 s->numberio = le64_to_cpu(s->numberio); 1447 for (i = 0; i < 4; i++) 1448 s->rand.s[i] = le32_to_cpu(s->rand.s[i]); 1449 for (i = 0; i < s->no_comps; i++) 1450 s->offsets[i] = le64_to_cpu(s->offsets[i]); 1451 1452 td->vstate = s; 1453} 1454 1455int verify_state_hdr(struct verify_state_hdr *hdr, struct thread_io_list *s) 1456{ 1457 uint64_t crc; 1458 1459 hdr->version = le64_to_cpu(hdr->version); 1460 hdr->size = le64_to_cpu(hdr->size); 1461 hdr->crc = le64_to_cpu(hdr->crc); 1462 1463 if (hdr->version != VSTATE_HDR_VERSION) 1464 return 1; 1465 1466 crc = fio_crc32c((void *)s, hdr->size); 1467 if (crc != hdr->crc) 1468 return 1; 1469 1470 return 0; 1471} 1472 1473int verify_load_state(struct thread_data *td, const char *prefix) 1474{ 1475 struct thread_io_list *s = NULL; 1476 struct verify_state_hdr hdr; 1477 uint64_t crc; 1478 ssize_t ret; 1479 int fd; 1480 1481 if (!td->o.verify_state) 1482 return 0; 1483 1484 fd = open_state_file(td->o.name, prefix, td->thread_number - 1, 0); 1485 if (fd == -1) 1486 return 1; 1487 1488 ret = read(fd, &hdr, sizeof(hdr)); 1489 if (ret != sizeof(hdr)) { 1490 if (ret < 0) 1491 td_verror(td, errno, "read verify state hdr"); 1492 log_err("fio: failed reading verify state header\n"); 1493 goto err; 1494 } 1495 1496 hdr.version = le64_to_cpu(hdr.version); 1497 hdr.size = le64_to_cpu(hdr.size); 1498 hdr.crc = le64_to_cpu(hdr.crc); 1499 1500 if (hdr.version != VSTATE_HDR_VERSION) { 1501 log_err("fio: bad version in verify state header\n"); 1502 goto err; 1503 } 1504 1505 s = malloc(hdr.size); 1506 ret = read(fd, s, hdr.size); 1507 if (ret != hdr.size) { 1508 if (ret < 0) 1509 td_verror(td, errno, "read verify state"); 1510 log_err("fio: failed reading verity state\n"); 1511 goto err; 1512 } 1513 1514 crc = fio_crc32c((void *)s, hdr.size); 1515 if (crc != hdr.crc) { 1516 log_err("fio: verify state is corrupt\n"); 1517 goto err; 1518 } 1519 1520 close(fd); 1521 1522 verify_convert_assign_state(td, s); 1523 return 0; 1524err: 1525 if (s) 1526 free(s); 1527 close(fd); 1528 return 1; 1529} 1530 1531/* 1532 * Use the loaded verify state to know when to stop doing verification 1533 */ 1534int verify_state_should_stop(struct thread_data *td, struct io_u *io_u) 1535{ 1536 struct thread_io_list *s = td->vstate; 1537 int i; 1538 1539 if (!s) 1540 return 0; 1541 1542 /* 1543 * If we're not into the window of issues - depth yet, continue. If 1544 * issue is shorter than depth, do check. 1545 */ 1546 if ((td->io_blocks[DDIR_READ] < s->depth || 1547 s->numberio - td->io_blocks[DDIR_READ] > s->depth) && 1548 s->numberio > s->depth) 1549 return 0; 1550 1551 /* 1552 * We're in the window of having to check if this io was 1553 * completed or not. If the IO was seen as completed, then 1554 * lets verify it. 1555 */ 1556 for (i = 0; i < s->no_comps; i++) 1557 if (io_u->offset == s->offsets[i]) 1558 return 0; 1559 1560 /* 1561 * Not found, we have to stop 1562 */ 1563 return 1; 1564} 1565